slatedb 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +150 -44
- data/ext/slatedb/Cargo.toml +7 -8
- data/ext/slatedb/src/checkpoint.rs +47 -0
- data/ext/slatedb/src/database.rs +61 -19
- data/ext/slatedb/src/lib.rs +2 -0
- data/ext/slatedb/src/merge_ops.rs +2 -185
- data/ext/slatedb/src/reader.rs +21 -6
- data/ext/slatedb/src/runtime.rs +1 -52
- data/ext/slatedb/src/snapshot.rs +4 -0
- data/ext/slatedb/src/transaction.rs +4 -0
- data/lib/slatedb/database.rb +51 -27
- data/lib/slatedb/reader.rb +38 -27
- data/lib/slatedb/snapshot.rb +17 -9
- data/lib/slatedb/transaction.rb +17 -9
- data/lib/slatedb/version.rb +1 -1
- metadata +6 -5
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
use std::sync::Arc;
|
|
2
|
-
use std::thread;
|
|
3
2
|
|
|
4
3
|
use bytes::Bytes;
|
|
5
|
-
use
|
|
6
|
-
use magnus::rb_sys::{AsRawValue, FromRawValue};
|
|
7
|
-
use magnus::value::ReprValue;
|
|
8
|
-
use magnus::{Error, RHash, Ruby, Value};
|
|
4
|
+
use magnus::{Error, RHash};
|
|
9
5
|
use slatedb::{MergeOperator, MergeOperatorError};
|
|
10
6
|
|
|
11
7
|
use crate::errors::invalid_argument_error;
|
|
@@ -39,166 +35,6 @@ impl MergeOperator for StringConcatMergeOperator {
|
|
|
39
35
|
}
|
|
40
36
|
}
|
|
41
37
|
|
|
42
|
-
/// A merge operator that calls a Ruby block/proc.
|
|
43
|
-
///
|
|
44
|
-
/// This stores the raw Ruby VALUE and calls it via `with_gvl` when merge
|
|
45
|
-
/// operations are needed. The proc is called with (key, existing_value, new_value)
|
|
46
|
-
/// and should return the merged value as a String.
|
|
47
|
-
///
|
|
48
|
-
/// # Thread Safety
|
|
49
|
-
///
|
|
50
|
-
/// The Ruby proc can only be called from the Ruby thread that created this operator.
|
|
51
|
-
/// If the merge is called from a different thread (e.g., a Tokio worker thread during
|
|
52
|
-
/// background compaction), the merge will use a fallback string concatenation behavior.
|
|
53
|
-
///
|
|
54
|
-
/// # Safety
|
|
55
|
-
///
|
|
56
|
-
/// The Ruby proc must be kept alive (not garbage collected) for the lifetime
|
|
57
|
-
/// of this operator. This is typically handled by storing a reference to the
|
|
58
|
-
/// proc in the Ruby Database object.
|
|
59
|
-
pub struct RubyProcMergeOperator {
|
|
60
|
-
/// The raw Ruby VALUE of the proc. We store this as a raw value because
|
|
61
|
-
/// magnus::Value is not Send+Sync, but we need to be thread-safe.
|
|
62
|
-
/// We re-acquire the GVL before using it, which makes this safe.
|
|
63
|
-
proc_value: usize,
|
|
64
|
-
/// The thread ID of the Ruby thread that created this operator.
|
|
65
|
-
/// We can only safely call Ruby from this thread.
|
|
66
|
-
ruby_thread_id: thread::ThreadId,
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
// SAFETY: We only access the proc_value when we hold the GVL via the Ruby thread,
|
|
70
|
-
// which ensures thread-safe access to Ruby objects.
|
|
71
|
-
unsafe impl Send for RubyProcMergeOperator {}
|
|
72
|
-
unsafe impl Sync for RubyProcMergeOperator {}
|
|
73
|
-
|
|
74
|
-
impl RubyProcMergeOperator {
|
|
75
|
-
/// Create a new RubyProcMergeOperator from a Ruby proc/block.
|
|
76
|
-
///
|
|
77
|
-
/// # Safety
|
|
78
|
-
///
|
|
79
|
-
/// The caller must ensure the proc remains alive (not GC'd) for the
|
|
80
|
-
/// lifetime of this operator.
|
|
81
|
-
pub fn new(proc: Value) -> Self {
|
|
82
|
-
Self {
|
|
83
|
-
proc_value: proc.as_raw() as usize,
|
|
84
|
-
ruby_thread_id: thread::current().id(),
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
/// Check if we're on the Ruby thread that created this operator.
|
|
89
|
-
fn is_ruby_thread(&self) -> bool {
|
|
90
|
-
thread::current().id() == self.ruby_thread_id
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
/// Call the Ruby proc with the given arguments.
|
|
94
|
-
/// This must only be called from the Ruby thread (after checking is_ruby_thread).
|
|
95
|
-
fn call_proc_on_ruby_thread(
|
|
96
|
-
&self,
|
|
97
|
-
key: &str,
|
|
98
|
-
existing_value: Option<&str>,
|
|
99
|
-
new_value: &str,
|
|
100
|
-
) -> Result<Bytes, MergeOperatorError> {
|
|
101
|
-
// We're on the Ruby thread, so we can use with_gvl
|
|
102
|
-
// Import here to avoid the circular dependency at module level
|
|
103
|
-
use crate::runtime::with_gvl;
|
|
104
|
-
|
|
105
|
-
let key_owned = key.to_string();
|
|
106
|
-
let existing_owned = existing_value.map(|s| s.to_string());
|
|
107
|
-
let new_owned = new_value.to_string();
|
|
108
|
-
|
|
109
|
-
with_gvl(|| {
|
|
110
|
-
let ruby = Ruby::get().expect("Ruby runtime not available");
|
|
111
|
-
|
|
112
|
-
// Reconstruct the proc Value from the raw pointer
|
|
113
|
-
let proc = unsafe { Value::from_raw(self.proc_value as _) };
|
|
114
|
-
|
|
115
|
-
// Build arguments: (key, existing_value, new_value)
|
|
116
|
-
let existing_arg: Value = match &existing_owned {
|
|
117
|
-
Some(s) => ruby.str_new(s).as_value(),
|
|
118
|
-
None => ruby.qnil().as_value(),
|
|
119
|
-
};
|
|
120
|
-
|
|
121
|
-
// Call the proc
|
|
122
|
-
let result: Result<String, magnus::Error> = proc.funcall(
|
|
123
|
-
"call",
|
|
124
|
-
(
|
|
125
|
-
ruby.str_new(&key_owned),
|
|
126
|
-
existing_arg,
|
|
127
|
-
ruby.str_new(&new_owned),
|
|
128
|
-
),
|
|
129
|
-
);
|
|
130
|
-
|
|
131
|
-
match result {
|
|
132
|
-
Ok(merged) => Ok(Bytes::from(merged)),
|
|
133
|
-
Err(e) => {
|
|
134
|
-
error!("Ruby merge operator error: {}", e);
|
|
135
|
-
Err(MergeOperatorError::EmptyBatch)
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
})
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/// Fallback merge when we're not on the Ruby thread.
|
|
142
|
-
/// Uses simple concatenation as a safe default.
|
|
143
|
-
fn fallback_merge(
|
|
144
|
-
&self,
|
|
145
|
-
existing_value: Option<&Bytes>,
|
|
146
|
-
new_value: &Bytes,
|
|
147
|
-
) -> Result<Bytes, MergeOperatorError> {
|
|
148
|
-
error!(
|
|
149
|
-
"Ruby merge operator called from non-Ruby thread, using fallback concatenation. \
|
|
150
|
-
This can happen during background compaction."
|
|
151
|
-
);
|
|
152
|
-
let mut result = existing_value.map(|v| v.to_vec()).unwrap_or_default();
|
|
153
|
-
result.extend_from_slice(new_value);
|
|
154
|
-
Ok(Bytes::from(result))
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
/// Call the Ruby proc with the given arguments, handling thread safety.
|
|
158
|
-
fn call_proc(
|
|
159
|
-
&self,
|
|
160
|
-
key: &Bytes,
|
|
161
|
-
existing_value: Option<&Bytes>,
|
|
162
|
-
new_value: &Bytes,
|
|
163
|
-
) -> Result<Bytes, MergeOperatorError> {
|
|
164
|
-
let key_str = String::from_utf8_lossy(key);
|
|
165
|
-
let existing_str = existing_value.map(|v| String::from_utf8_lossy(v));
|
|
166
|
-
let new_str = String::from_utf8_lossy(new_value);
|
|
167
|
-
|
|
168
|
-
if self.is_ruby_thread() {
|
|
169
|
-
self.call_proc_on_ruby_thread(&key_str, existing_str.as_deref(), &new_str)
|
|
170
|
-
} else {
|
|
171
|
-
// We're on a worker thread, use fallback
|
|
172
|
-
self.fallback_merge(existing_value, new_value)
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
impl MergeOperator for RubyProcMergeOperator {
|
|
178
|
-
fn merge(
|
|
179
|
-
&self,
|
|
180
|
-
key: &Bytes,
|
|
181
|
-
existing_value: Option<Bytes>,
|
|
182
|
-
value: Bytes,
|
|
183
|
-
) -> Result<Bytes, MergeOperatorError> {
|
|
184
|
-
self.call_proc(key, existing_value.as_ref(), &value)
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
fn merge_batch(
|
|
188
|
-
&self,
|
|
189
|
-
key: &Bytes,
|
|
190
|
-
existing_value: Option<Bytes>,
|
|
191
|
-
operands: &[Bytes],
|
|
192
|
-
) -> Result<Bytes, MergeOperatorError> {
|
|
193
|
-
// Apply operands one at a time through the Ruby proc
|
|
194
|
-
let mut current = existing_value;
|
|
195
|
-
for operand in operands {
|
|
196
|
-
current = Some(self.call_proc(key, current.as_ref(), operand)?);
|
|
197
|
-
}
|
|
198
|
-
Ok(current.unwrap_or_default())
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
|
|
202
38
|
pub fn parse_merge_operator(
|
|
203
39
|
kwargs: &RHash,
|
|
204
40
|
) -> Result<Option<Arc<dyn MergeOperator + Send + Sync>>, Error> {
|
|
@@ -211,7 +47,7 @@ pub fn parse_merge_operator(
|
|
|
211
47
|
"string_concat" | "concat" => Arc::new(StringConcatMergeOperator),
|
|
212
48
|
_ => {
|
|
213
49
|
return Err(invalid_argument_error(&format!(
|
|
214
|
-
"invalid merge_operator: {} (expected 'string_concat'
|
|
50
|
+
"invalid merge_operator: {} (expected 'string_concat' or 'concat')",
|
|
215
51
|
merge_operator
|
|
216
52
|
)))
|
|
217
53
|
}
|
|
@@ -219,22 +55,3 @@ pub fn parse_merge_operator(
|
|
|
219
55
|
|
|
220
56
|
Ok(Some(operator))
|
|
221
57
|
}
|
|
222
|
-
|
|
223
|
-
/// Parse a Ruby proc as a merge operator.
|
|
224
|
-
pub fn parse_merge_operator_proc(
|
|
225
|
-
kwargs: &RHash,
|
|
226
|
-
) -> Result<Option<Arc<dyn MergeOperator + Send + Sync>>, Error> {
|
|
227
|
-
let proc_value = get_optional::<Value>(kwargs, "merge_operator_proc")?;
|
|
228
|
-
let Some(proc) = proc_value else {
|
|
229
|
-
return Ok(None);
|
|
230
|
-
};
|
|
231
|
-
|
|
232
|
-
// Verify it's callable
|
|
233
|
-
if !proc.respond_to("call", false).unwrap_or(false) {
|
|
234
|
-
return Err(invalid_argument_error(
|
|
235
|
-
"merge_operator_proc must respond to 'call'",
|
|
236
|
-
));
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
Ok(Some(Arc::new(RubyProcMergeOperator::new(proc))))
|
|
240
|
-
}
|
data/ext/slatedb/src/reader.rs
CHANGED
|
@@ -9,6 +9,7 @@ use slatedb::IterationOrder;
|
|
|
9
9
|
|
|
10
10
|
use crate::errors::invalid_argument_error;
|
|
11
11
|
use crate::iterator::Iterator;
|
|
12
|
+
use crate::merge_ops::parse_merge_operator;
|
|
12
13
|
use crate::runtime::block_on_result;
|
|
13
14
|
use crate::utils::{get_optional, prefix_subrange_from_kwargs, resolve_object_store};
|
|
14
15
|
|
|
@@ -57,6 +58,7 @@ impl Reader {
|
|
|
57
58
|
let max_open_file_handles = get_optional::<usize>(&kwargs, "max_open_file_handles")?;
|
|
58
59
|
let cache_root = get_optional::<String>(&kwargs, "cache_root")?;
|
|
59
60
|
let follow_latest = get_optional::<bool>(&kwargs, "follow_latest")?.unwrap_or(false);
|
|
61
|
+
let merge_operator = parse_merge_operator(&kwargs)?;
|
|
60
62
|
|
|
61
63
|
// Resolve the reader mode from checkpoint_id / follow_latest.
|
|
62
64
|
let mode = match checkpoint_id {
|
|
@@ -96,14 +98,23 @@ impl Reader {
|
|
|
96
98
|
if let Some(skip_replay) = skip_wal_replay {
|
|
97
99
|
options.skip_wal_replay = skip_replay;
|
|
98
100
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
101
|
+
#[allow(deprecated)]
|
|
102
|
+
{
|
|
103
|
+
if let Some(ref root) = cache_root {
|
|
104
|
+
options.object_store_cache_options.root_folder =
|
|
105
|
+
Some(std::path::PathBuf::from(root));
|
|
106
|
+
}
|
|
107
|
+
if let Some(max_handles) = max_open_file_handles {
|
|
108
|
+
options.object_store_cache_options.max_open_file_handles = max_handles;
|
|
109
|
+
}
|
|
102
110
|
}
|
|
103
|
-
|
|
104
|
-
|
|
111
|
+
let mut builder = DbReader::builder(path, object_store)
|
|
112
|
+
.with_reader_mode(mode)
|
|
113
|
+
.with_options(options);
|
|
114
|
+
if let Some(operator) = merge_operator {
|
|
115
|
+
builder = builder.with_merge_operator(operator);
|
|
105
116
|
}
|
|
106
|
-
|
|
117
|
+
builder.build().await
|
|
107
118
|
})?;
|
|
108
119
|
|
|
109
120
|
Ok(Self {
|
|
@@ -150,6 +161,10 @@ impl Reader {
|
|
|
150
161
|
opts.cache_blocks = cb;
|
|
151
162
|
}
|
|
152
163
|
|
|
164
|
+
if let Some(limit) = get_optional::<usize>(&kwargs, "max_concurrent_sources")? {
|
|
165
|
+
opts.max_concurrent_sources = limit;
|
|
166
|
+
}
|
|
167
|
+
|
|
153
168
|
let result =
|
|
154
169
|
block_on_result(async { self.inner.get_with_options(key.as_bytes(), &opts).await })?;
|
|
155
170
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
data/ext/slatedb/src/runtime.rs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
use magnus::Error;
|
|
2
2
|
use once_cell::sync::OnceCell;
|
|
3
|
-
use rb_sys::
|
|
3
|
+
use rb_sys::rb_thread_call_without_gvl;
|
|
4
4
|
use slatedb::Error as SlateError;
|
|
5
5
|
use std::ffi::c_void;
|
|
6
6
|
use std::future::Future;
|
|
@@ -98,54 +98,3 @@ where
|
|
|
98
98
|
|
|
99
99
|
closure.result.expect("closure did not run")
|
|
100
100
|
}
|
|
101
|
-
|
|
102
|
-
/// Execute a closure while holding the Ruby GVL.
|
|
103
|
-
///
|
|
104
|
-
/// This is the inverse of `without_gvl`. Use this when you need to call back
|
|
105
|
-
/// into Ruby from code that has previously released the GVL (e.g., inside
|
|
106
|
-
/// a future being executed by `block_on`).
|
|
107
|
-
///
|
|
108
|
-
/// # Safety
|
|
109
|
-
///
|
|
110
|
-
/// This function can ONLY be called from a Ruby thread that has previously
|
|
111
|
-
/// released the GVL via `without_gvl` or `rb_thread_call_without_gvl`.
|
|
112
|
-
/// Calling it from a non-Ruby thread (like a spawned Tokio task) will cause
|
|
113
|
-
/// a Ruby fatal error.
|
|
114
|
-
///
|
|
115
|
-
/// # Panics
|
|
116
|
-
///
|
|
117
|
-
/// Panics if called from a non-Ruby thread.
|
|
118
|
-
pub fn with_gvl<F, T>(f: F) -> T
|
|
119
|
-
where
|
|
120
|
-
F: FnOnce() -> T,
|
|
121
|
-
{
|
|
122
|
-
struct Closure<F, T> {
|
|
123
|
-
f: Option<F>,
|
|
124
|
-
result: Option<T>,
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
extern "C" fn call_closure<F, T>(data: *mut c_void) -> *mut c_void
|
|
128
|
-
where
|
|
129
|
-
F: FnOnce() -> T,
|
|
130
|
-
{
|
|
131
|
-
let closure = unsafe { &mut *(data as *mut Closure<F, T>) };
|
|
132
|
-
if let Some(f) = closure.f.take() {
|
|
133
|
-
closure.result = Some(f());
|
|
134
|
-
}
|
|
135
|
-
std::ptr::null_mut()
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
let mut closure = Closure {
|
|
139
|
-
f: Some(f),
|
|
140
|
-
result: None,
|
|
141
|
-
};
|
|
142
|
-
|
|
143
|
-
unsafe {
|
|
144
|
-
rb_thread_call_with_gvl(
|
|
145
|
-
Some(call_closure::<F, T>),
|
|
146
|
-
&mut closure as *mut _ as *mut c_void,
|
|
147
|
-
);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
closure.result.expect("closure did not run")
|
|
151
|
-
}
|
data/ext/slatedb/src/snapshot.rs
CHANGED
|
@@ -73,6 +73,10 @@ impl Snapshot {
|
|
|
73
73
|
opts.cache_blocks = cb;
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
+
if let Some(limit) = get_optional::<usize>(&kwargs, "max_concurrent_sources")? {
|
|
77
|
+
opts.max_concurrent_sources = limit;
|
|
78
|
+
}
|
|
79
|
+
|
|
76
80
|
let guard = self.inner.borrow();
|
|
77
81
|
let snapshot = guard
|
|
78
82
|
.as_ref()
|
data/lib/slatedb/database.rb
CHANGED
|
@@ -9,9 +9,18 @@ module SlateDb
|
|
|
9
9
|
#
|
|
10
10
|
# @param path [String] The path identifier for the database
|
|
11
11
|
# @param url [String, nil] Optional object store URL (e.g., "s3://bucket/path")
|
|
12
|
-
# @param merge_operator [Symbol, String,
|
|
13
|
-
#
|
|
14
|
-
#
|
|
12
|
+
# @param merge_operator [Symbol, String, nil] Optional built-in merge operator
|
|
13
|
+
# (:string_concat or :concat). Ruby callbacks are not supported.
|
|
14
|
+
# @param compactor_checkpoint_lifetime [Integer, nil] How long compaction inputs
|
|
15
|
+
# are protected from garbage collection, in milliseconds (default: 15 minutes).
|
|
16
|
+
# Reads using those inputs must finish within this time.
|
|
17
|
+
# @param sorted_run_consolidation_threshold [Integer, nil] Sorted-run count above
|
|
18
|
+
# which fallback consolidation starts. Zero disables it (default); use at least
|
|
19
|
+
# two to enable it. One is disabled by SlateDB with a warning. This is not a hard limit.
|
|
20
|
+
# @param experimental_incremental_compaction [Boolean, nil] Publish sorted-run
|
|
21
|
+
# compaction output before completion (experimental, disabled by default).
|
|
22
|
+
# @param sst_block_alignment [Boolean, nil] Align SST data blocks to 4 KiB
|
|
23
|
+
# boundaries (disabled by default; incompatible with compression).
|
|
15
24
|
# @yield [db] If a block is given, yields the database and ensures it's closed
|
|
16
25
|
# @return [Database] The opened database (or block result if block given)
|
|
17
26
|
#
|
|
@@ -28,28 +37,22 @@ module SlateDb
|
|
|
28
37
|
# @example Open with S3 backend
|
|
29
38
|
# db = SlateDb::Database.open("/tmp/mydb", url: "s3://mybucket/path")
|
|
30
39
|
#
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
case merge_operator
|
|
45
|
-
when Symbol, String
|
|
46
|
-
opts[:merge_operator] = merge_operator.to_s
|
|
47
|
-
when Proc
|
|
48
|
-
# Store the proc to prevent GC and pass to Rust
|
|
49
|
-
@_merge_operator_proc = merge_operator
|
|
50
|
-
opts[:merge_operator_proc] = merge_operator
|
|
40
|
+
def open(path, url: nil, merge_operator: nil,
|
|
41
|
+
compactor_checkpoint_lifetime: nil, sorted_run_consolidation_threshold: nil,
|
|
42
|
+
experimental_incremental_compaction: nil, sst_block_alignment: nil)
|
|
43
|
+
opts = {
|
|
44
|
+
compactor_checkpoint_lifetime: compactor_checkpoint_lifetime,
|
|
45
|
+
sorted_run_consolidation_threshold: sorted_run_consolidation_threshold,
|
|
46
|
+
experimental_incremental_compaction: experimental_incremental_compaction,
|
|
47
|
+
sst_block_alignment: sst_block_alignment
|
|
48
|
+
}.compact
|
|
49
|
+
|
|
50
|
+
unless merge_operator.nil? || merge_operator.is_a?(Symbol) || merge_operator.is_a?(String)
|
|
51
|
+
raise ArgumentError, "merge_operator must be :string_concat or :concat; Ruby callbacks are not supported"
|
|
51
52
|
end
|
|
52
53
|
|
|
54
|
+
opts[:merge_operator] = merge_operator.to_s if merge_operator
|
|
55
|
+
|
|
53
56
|
db = _open(path, url, opts)
|
|
54
57
|
|
|
55
58
|
if block_given?
|
|
@@ -74,6 +77,8 @@ module SlateDb
|
|
|
74
77
|
# @param durability_filter [String, nil] Filter by durability level ("remote" or "memory")
|
|
75
78
|
# @param dirty [Boolean, nil] Whether to include uncommitted data
|
|
76
79
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
80
|
+
# @param max_concurrent_sources [Integer, nil] Concurrent sources to fetch after
|
|
81
|
+
# the first source misses (default: 1; zero also means serial fetching).
|
|
77
82
|
# @return [String, nil] The value, or nil if not found
|
|
78
83
|
#
|
|
79
84
|
# @example Basic get
|
|
@@ -82,11 +87,12 @@ module SlateDb
|
|
|
82
87
|
# @example Get with options
|
|
83
88
|
# value = db.get("mykey", durability_filter: "memory", dirty: true)
|
|
84
89
|
#
|
|
85
|
-
def get(key, durability_filter: nil, dirty: nil, cache_blocks: nil)
|
|
90
|
+
def get(key, durability_filter: nil, dirty: nil, cache_blocks: nil, max_concurrent_sources: nil)
|
|
86
91
|
opts = {}
|
|
87
92
|
opts[:durability_filter] = durability_filter.to_s if durability_filter
|
|
88
93
|
opts[:dirty] = dirty unless dirty.nil?
|
|
89
94
|
opts[:cache_blocks] = cache_blocks unless cache_blocks.nil?
|
|
95
|
+
opts[:max_concurrent_sources] = max_concurrent_sources unless max_concurrent_sources.nil?
|
|
90
96
|
|
|
91
97
|
if opts.empty?
|
|
92
98
|
_get(key)
|
|
@@ -102,17 +108,20 @@ module SlateDb
|
|
|
102
108
|
# @param dirty [Boolean, nil] Whether to include uncommitted data
|
|
103
109
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
104
110
|
# @return [Hash, nil] A hash with :key, :value, :seq, :create_ts, and :expire_ts, or nil if not found
|
|
111
|
+
# @param max_concurrent_sources [Integer, nil] Concurrent sources after the first
|
|
112
|
+
# miss. See {#get}.
|
|
105
113
|
#
|
|
106
114
|
# @example Inspect metadata
|
|
107
115
|
# entry = db.get_key_value("mykey")
|
|
108
116
|
# entry[:value] # => "myvalue"
|
|
109
117
|
# entry[:seq] # => sequence number
|
|
110
118
|
#
|
|
111
|
-
def get_key_value(key, durability_filter: nil, dirty: nil, cache_blocks: nil)
|
|
119
|
+
def get_key_value(key, durability_filter: nil, dirty: nil, cache_blocks: nil, max_concurrent_sources: nil)
|
|
112
120
|
opts = {}
|
|
113
121
|
opts[:durability_filter] = durability_filter.to_s if durability_filter
|
|
114
122
|
opts[:dirty] = dirty unless dirty.nil?
|
|
115
123
|
opts[:cache_blocks] = cache_blocks unless cache_blocks.nil?
|
|
124
|
+
opts[:max_concurrent_sources] = max_concurrent_sources unless max_concurrent_sources.nil?
|
|
116
125
|
|
|
117
126
|
if opts.empty?
|
|
118
127
|
_get_key_value(key)
|
|
@@ -484,6 +493,8 @@ module SlateDb
|
|
|
484
493
|
#
|
|
485
494
|
# @param lifetime [Integer, nil] Checkpoint lifetime in milliseconds
|
|
486
495
|
# @param name [String, nil] Optional name for the checkpoint
|
|
496
|
+
# @param scope [Symbol, String] :durable (default) captures durable writes;
|
|
497
|
+
# :all also flushes pending writes before storing the checkpoint.
|
|
487
498
|
# @return [Hash] Hash with :id (UUID string) and :manifest_id (integer)
|
|
488
499
|
#
|
|
489
500
|
# @example Create a named checkpoint
|
|
@@ -493,13 +504,26 @@ module SlateDb
|
|
|
493
504
|
# @example Create a checkpoint with lifetime
|
|
494
505
|
# checkpoint = db.create_checkpoint(lifetime: 3600_000) # 1 hour
|
|
495
506
|
#
|
|
496
|
-
def create_checkpoint(lifetime: nil, name: nil)
|
|
497
|
-
opts = {}
|
|
507
|
+
def create_checkpoint(lifetime: nil, name: nil, scope: :durable)
|
|
508
|
+
opts = { scope: scope.to_s }
|
|
498
509
|
opts[:lifetime] = lifetime if lifetime
|
|
499
510
|
opts[:name] = name if name
|
|
500
511
|
_create_checkpoint(opts)
|
|
501
512
|
end
|
|
502
513
|
|
|
514
|
+
# Start a checkpoint without waiting for it to be stored.
|
|
515
|
+
#
|
|
516
|
+
# With scope: :all, captures writes completed before this call and excludes
|
|
517
|
+
# writes started after it returns. Writes during the call may be included.
|
|
518
|
+
# Options match {#create_checkpoint}. Dropping the handle does not cancel it.
|
|
519
|
+
# @return [CheckpointHandle] #id is available immediately; #wait returns the
|
|
520
|
+
# :id/:manifest_id hash once stored and consumes the handle. A second wait
|
|
521
|
+
# raises ClosedError. #id remains available after waiting.
|
|
522
|
+
def begin_checkpoint(lifetime: nil, name: nil, scope: :durable)
|
|
523
|
+
opts = { scope: scope.to_s, lifetime: lifetime, name: name }.compact
|
|
524
|
+
_begin_checkpoint(opts)
|
|
525
|
+
end
|
|
526
|
+
|
|
503
527
|
# Get database metrics registry.
|
|
504
528
|
#
|
|
505
529
|
# @return [Metrics] Metrics registry
|
data/lib/slatedb/reader.rb
CHANGED
|
@@ -21,6 +21,7 @@ module SlateDb
|
|
|
21
21
|
# @param cache_root [String, nil] Root folder for the reader's local on-disk
|
|
22
22
|
# object-store cache. Setting this enables the cached object store; when it is
|
|
23
23
|
# not set the cache (and `max_open_file_handles`) has no effect.
|
|
24
|
+
# This legacy cache is deprecated upstream in SlateDB 0.18.1.
|
|
24
25
|
# @param max_open_file_handles [Integer, nil] Maximum number of file handles to keep
|
|
25
26
|
# open in the reader's file-handle cache. When the limit is reached, the least
|
|
26
27
|
# recently used handle is closed (default: 1000). Only takes effect when
|
|
@@ -54,29 +55,31 @@ module SlateDb
|
|
|
54
55
|
manifest_poll_interval: nil, checkpoint_lifetime: nil,
|
|
55
56
|
max_memtable_bytes: nil, cache_root: nil, max_open_file_handles: nil,
|
|
56
57
|
merge_operator: nil)
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
opts
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
58
|
+
unless merge_operator.nil? || merge_operator.is_a?(Symbol) || merge_operator.is_a?(String)
|
|
59
|
+
raise ArgumentError, "merge_operator must be :string_concat or :concat; Ruby callbacks are not supported"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
opts = {
|
|
63
|
+
follow_latest: follow_latest,
|
|
64
|
+
manifest_poll_interval: manifest_poll_interval,
|
|
65
|
+
checkpoint_lifetime: checkpoint_lifetime,
|
|
66
|
+
max_memtable_bytes: max_memtable_bytes,
|
|
67
|
+
cache_root: cache_root,
|
|
68
|
+
max_open_file_handles: max_open_file_handles,
|
|
69
|
+
merge_operator: merge_operator&.to_s
|
|
70
|
+
}.compact
|
|
65
71
|
|
|
66
72
|
reader = _open(path, url, checkpoint_id, opts)
|
|
73
|
+
return reader unless block_given?
|
|
67
74
|
|
|
68
|
-
|
|
75
|
+
begin
|
|
76
|
+
yield reader
|
|
77
|
+
ensure
|
|
69
78
|
begin
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
reader.close
|
|
74
|
-
rescue StandardError
|
|
75
|
-
nil
|
|
76
|
-
end
|
|
79
|
+
reader.close
|
|
80
|
+
rescue StandardError
|
|
81
|
+
nil
|
|
77
82
|
end
|
|
78
|
-
else
|
|
79
|
-
reader
|
|
80
83
|
end
|
|
81
84
|
end
|
|
82
85
|
end
|
|
@@ -87,13 +90,16 @@ module SlateDb
|
|
|
87
90
|
# @param durability_filter [String, nil] Filter by durability level ("remote" or "memory")
|
|
88
91
|
# @param dirty [Boolean, nil] Whether to include uncommitted data
|
|
89
92
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
93
|
+
# @param max_concurrent_sources [Integer, nil] Concurrent sources after the first
|
|
94
|
+
# miss (default: 1; zero also means serial fetching).
|
|
90
95
|
# @return [String, nil] The value, or nil if not found
|
|
91
96
|
#
|
|
92
|
-
def get(key, durability_filter: nil, dirty: nil, cache_blocks: nil)
|
|
97
|
+
def get(key, durability_filter: nil, dirty: nil, cache_blocks: nil, max_concurrent_sources: nil)
|
|
93
98
|
opts = {}
|
|
94
99
|
opts[:durability_filter] = durability_filter.to_s if durability_filter
|
|
95
100
|
opts[:dirty] = dirty unless dirty.nil?
|
|
96
101
|
opts[:cache_blocks] = cache_blocks unless cache_blocks.nil?
|
|
102
|
+
opts[:max_concurrent_sources] = max_concurrent_sources unless max_concurrent_sources.nil?
|
|
97
103
|
|
|
98
104
|
if opts.empty?
|
|
99
105
|
_get(key)
|
|
@@ -106,16 +112,19 @@ module SlateDb
|
|
|
106
112
|
#
|
|
107
113
|
# @param start_key [String] The start key (inclusive)
|
|
108
114
|
# @param end_key [String, nil] The end key (exclusive)
|
|
115
|
+
# @param order [Symbol, String, nil] Iteration order (:asc/:ascending or :desc/:descending)
|
|
109
116
|
# @return [Iterator] An iterator over key-value pairs
|
|
110
117
|
#
|
|
111
118
|
def scan(start_key, end_key = nil, durability_filter: nil, dirty: nil,
|
|
112
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, &)
|
|
113
|
-
opts = {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, order: nil, &)
|
|
120
|
+
opts = {
|
|
121
|
+
durability_filter: durability_filter&.to_s,
|
|
122
|
+
dirty: dirty,
|
|
123
|
+
read_ahead_bytes: read_ahead_bytes,
|
|
124
|
+
cache_blocks: cache_blocks,
|
|
125
|
+
max_fetch_tasks: max_fetch_tasks,
|
|
126
|
+
order: order&.to_s
|
|
127
|
+
}.compact
|
|
119
128
|
|
|
120
129
|
iter = if opts.empty?
|
|
121
130
|
_scan(start_key, end_key)
|
|
@@ -138,6 +147,7 @@ module SlateDb
|
|
|
138
147
|
# @param read_ahead_bytes [Integer, nil] Number of bytes to read ahead
|
|
139
148
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
140
149
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
150
|
+
# @param order [Symbol, String, nil] Iteration order (:asc/:ascending or :desc/:descending)
|
|
141
151
|
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
142
152
|
# prefix, to start scanning from (e.g. prefix "user:" with from "100"
|
|
143
153
|
# starts at "user:100"). Defaults to the start of the prefix.
|
|
@@ -146,7 +156,7 @@ module SlateDb
|
|
|
146
156
|
# @return [Iterator] An iterator over key-value pairs
|
|
147
157
|
#
|
|
148
158
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
149
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
159
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, order: nil,
|
|
150
160
|
from: nil, to: nil, &)
|
|
151
161
|
opts = {
|
|
152
162
|
durability_filter: durability_filter&.to_s,
|
|
@@ -154,6 +164,7 @@ module SlateDb
|
|
|
154
164
|
read_ahead_bytes: read_ahead_bytes,
|
|
155
165
|
cache_blocks: cache_blocks,
|
|
156
166
|
max_fetch_tasks: max_fetch_tasks,
|
|
167
|
+
order: order&.to_s,
|
|
157
168
|
subrange_from: from,
|
|
158
169
|
subrange_to: to
|
|
159
170
|
}.compact
|