slatedb 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +38 -25
- data/ext/slatedb/Cargo.toml +2 -1
- data/ext/slatedb/src/admin.rs +14 -20
- data/ext/slatedb/src/database.rs +34 -49
- data/ext/slatedb/src/iterator.rs +41 -22
- data/ext/slatedb/src/reader.rs +17 -24
- data/ext/slatedb/src/runtime.rs +28 -6
- data/ext/slatedb/src/snapshot.rs +11 -15
- data/ext/slatedb/src/transaction.rs +12 -18
- data/ext/slatedb/src/utils.rs +40 -0
- data/lib/slatedb/database.rb +2 -0
- data/lib/slatedb/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c93234ef6d9251d54b90cd816983c1d073d159587ae655fbef8e4e3b433928a3
|
|
4
|
+
data.tar.gz: 0b7aa7b9137ff2228f7e4153da2daafbb235ce857b51faa8d552c55f18cb7e7a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 54cc3a48576fc4ca07dc6ae95a755aa6fc5dd7fa7af46eba3452f3465c74d005d9f29d06ac1cc9ddf3461d602d910d63a08f55b0d486140ed22024f3ce1e3e4e
|
|
7
|
+
data.tar.gz: 2bd1c37ca9fcfa737bd47fd3d76b136376819b573a08e044c06114f5555d5243e8b465b7bbc8cba02f3461982cc5ea1e4195fb31e53a5f0d4dcd608d1ea79f55
|
data/README.md
CHANGED
|
@@ -2,35 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
Ruby bindings for [SlateDB](https://slatedb.io), a cloud-native embedded key-value store built on object storage.
|
|
4
4
|
|
|
5
|
-
[](https://buildkite.com/catkins-test/slatedb-rb)
|
|
5
|
+
[](https://buildkite.com/catkins-test/slatedb-rb) [](https://badge.fury.io/rb/slatedb)
|
|
6
6
|
|
|
7
|
-
##
|
|
7
|
+
## Production Readiness
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
These bindings are still in early development, and while SlateDB itself is used in Production, these bindings have yet to be. Contributions are welcome!
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
- All operations (get, put, delete, scan, transactions) are thread-safe
|
|
13
|
-
- The Ruby bindings release the Global VM Lock (GVL) during I/O operations, allowing other Ruby threads to run concurrently
|
|
14
|
-
- Perfect for use with multi-threaded Ruby applications like Puma, Sidekiq, and concurrent test suites
|
|
11
|
+
### TODO
|
|
15
12
|
|
|
16
|
-
|
|
17
|
-
db = SlateDb::Database.open("/tmp/mydb")
|
|
18
|
-
|
|
19
|
-
# Safe to use from multiple threads
|
|
20
|
-
threads = 10.times.map do |i|
|
|
21
|
-
Thread.new do
|
|
22
|
-
db.put("key-#{i}", "value-#{i}")
|
|
23
|
-
db.get("key-#{i}")
|
|
24
|
-
end
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
threads.each(&:join)
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
**Implementation details:**
|
|
31
|
-
- The underlying SlateDB library uses `Arc` (atomic reference counting) and `RwLock` for internal state management
|
|
32
|
-
- I/O operations release the Ruby GVL using `rb_thread_call_without_gvl`, preventing blocking other threads
|
|
33
|
-
- A shared Tokio multi-threaded runtime handles all async operations efficiently
|
|
13
|
+
- [ ] Cross-compile native extensions
|
|
34
14
|
|
|
35
15
|
## Installation
|
|
36
16
|
|
|
@@ -52,6 +32,9 @@ Or install it yourself as:
|
|
|
52
32
|
gem install slatedb
|
|
53
33
|
```
|
|
54
34
|
|
|
35
|
+
> [!IMPORTANT]
|
|
36
|
+
> This gem currently requires a working Rust toolchain to install until the dependencies are cross-compiled.
|
|
37
|
+
|
|
55
38
|
## Usage
|
|
56
39
|
|
|
57
40
|
### Basic Operations
|
|
@@ -347,6 +330,34 @@ db.put("key", "value")
|
|
|
347
330
|
db.flush
|
|
348
331
|
```
|
|
349
332
|
|
|
333
|
+
## Thread Safety
|
|
334
|
+
|
|
335
|
+
**SlateDB is fully thread-safe and optimized for concurrent access.**
|
|
336
|
+
|
|
337
|
+
- The `Database` class can be safely shared across multiple Ruby threads
|
|
338
|
+
- All operations (get, put, delete, scan, transactions) are thread-safe
|
|
339
|
+
- The Ruby bindings release the Global VM Lock (GVL) during I/O operations, allowing other Ruby threads to run concurrently
|
|
340
|
+
- Perfect for use with multi-threaded Ruby applications like Puma, Sidekiq, and concurrent test suites
|
|
341
|
+
|
|
342
|
+
```ruby
|
|
343
|
+
db = SlateDb::Database.open("/tmp/mydb")
|
|
344
|
+
|
|
345
|
+
# Safe to use from multiple threads
|
|
346
|
+
threads = 10.times.map do |i|
|
|
347
|
+
Thread.new do
|
|
348
|
+
db.put("key-#{i}", "value-#{i}")
|
|
349
|
+
db.get("key-#{i}")
|
|
350
|
+
end
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
threads.each(&:join)
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
**Implementation details:**
|
|
357
|
+
- The underlying SlateDB library uses `Arc` (atomic reference counting) and `RwLock` for internal state management
|
|
358
|
+
- I/O operations release the Ruby GVL using `rb_thread_call_without_gvl`, preventing blocking other threads
|
|
359
|
+
- A shared Tokio multi-threaded runtime handles all async operations efficiently
|
|
360
|
+
|
|
350
361
|
## Error Handling
|
|
351
362
|
|
|
352
363
|
SlateDB defines several exception classes:
|
|
@@ -399,6 +410,8 @@ bundle exec rspec spec/transaction_spec.rb
|
|
|
399
410
|
|
|
400
411
|
Bug reports and pull requests are welcome on GitHub at https://github.com/catkins/slatedb-rb.
|
|
401
412
|
|
|
413
|
+
Also, find me on the [SlateDB Discord Server](https://discord.gg/mHYmGy5MgA).
|
|
414
|
+
|
|
402
415
|
## License
|
|
403
416
|
|
|
404
417
|
Apache-2.0
|
data/ext/slatedb/Cargo.toml
CHANGED
|
@@ -16,7 +16,8 @@ magnus = { version = "0.8", features = ["rb-sys"] }
|
|
|
16
16
|
rb-sys = { version = "0.9", features = ["stable-api-compiled-fallback"] }
|
|
17
17
|
tokio = { version = "1", features = ["rt-multi-thread", "sync"] }
|
|
18
18
|
bytes = "1"
|
|
19
|
-
object_store = "0.12"
|
|
19
|
+
object_store = { version = "0.12", features = ["aws"] }
|
|
20
|
+
url = "2"
|
|
20
21
|
once_cell = "1"
|
|
21
22
|
log = "0.4"
|
|
22
23
|
uuid = "1"
|
data/ext/slatedb/src/admin.rs
CHANGED
|
@@ -5,9 +5,9 @@ use magnus::{function, method, Error, RHash, Ruby};
|
|
|
5
5
|
use slatedb::admin::AdminBuilder;
|
|
6
6
|
use slatedb::config::{CheckpointOptions, GarbageCollectorOptions};
|
|
7
7
|
|
|
8
|
-
use crate::errors::
|
|
9
|
-
use crate::runtime::block_on;
|
|
10
|
-
use crate::utils::get_optional;
|
|
8
|
+
use crate::errors::invalid_argument_error;
|
|
9
|
+
use crate::runtime::{block_on, block_on_result};
|
|
10
|
+
use crate::utils::{get_optional, resolve_object_store};
|
|
11
11
|
|
|
12
12
|
/// Ruby wrapper for SlateDB Admin.
|
|
13
13
|
///
|
|
@@ -25,16 +25,13 @@ impl Admin {
|
|
|
25
25
|
/// * `path` - The path identifier for the database
|
|
26
26
|
/// * `url` - Optional object store URL
|
|
27
27
|
pub fn new(path: String, url: Option<String>) -> Result<Self, Error> {
|
|
28
|
-
let
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
};
|
|
34
|
-
|
|
35
|
-
Ok::<_, Error>(AdminBuilder::new(path, object_store).build())
|
|
36
|
-
})?;
|
|
28
|
+
let object_store: Arc<dyn object_store::ObjectStore> = if let Some(ref url) = url {
|
|
29
|
+
block_on_result(async { resolve_object_store(url) })?
|
|
30
|
+
} else {
|
|
31
|
+
Arc::new(object_store::memory::InMemory::new())
|
|
32
|
+
};
|
|
37
33
|
|
|
34
|
+
let admin = AdminBuilder::new(path, object_store).build();
|
|
38
35
|
Ok(Self { inner: admin })
|
|
39
36
|
}
|
|
40
37
|
|
|
@@ -103,8 +100,8 @@ impl Admin {
|
|
|
103
100
|
name,
|
|
104
101
|
};
|
|
105
102
|
|
|
106
|
-
let result =
|
|
107
|
-
.
|
|
103
|
+
let result =
|
|
104
|
+
block_on_result(async { self.inner.create_detached_checkpoint(&options).await })?;
|
|
108
105
|
|
|
109
106
|
let ruby = Ruby::get().expect("Ruby runtime not available");
|
|
110
107
|
let hash = ruby.hash_new();
|
|
@@ -158,12 +155,11 @@ impl Admin {
|
|
|
158
155
|
|
|
159
156
|
let lifetime_duration = lifetime.map(std::time::Duration::from_millis);
|
|
160
157
|
|
|
161
|
-
|
|
158
|
+
block_on_result(async {
|
|
162
159
|
self.inner
|
|
163
160
|
.refresh_checkpoint(checkpoint_uuid, lifetime_duration)
|
|
164
161
|
.await
|
|
165
|
-
})
|
|
166
|
-
.map_err(map_error)?;
|
|
162
|
+
})?;
|
|
167
163
|
|
|
168
164
|
Ok(())
|
|
169
165
|
}
|
|
@@ -176,9 +172,7 @@ impl Admin {
|
|
|
176
172
|
let checkpoint_uuid = uuid::Uuid::parse_str(&id)
|
|
177
173
|
.map_err(|e| invalid_argument_error(&format!("invalid checkpoint UUID: {}", e)))?;
|
|
178
174
|
|
|
179
|
-
|
|
180
|
-
.map_err(map_error)?;
|
|
181
|
-
|
|
175
|
+
block_on_result(async { self.inner.delete_checkpoint(checkpoint_uuid).await })?;
|
|
182
176
|
Ok(())
|
|
183
177
|
}
|
|
184
178
|
|
data/ext/slatedb/src/database.rs
CHANGED
|
@@ -6,12 +6,12 @@ use slatedb::config::{DurabilityLevel, PutOptions, ReadOptions, ScanOptions, Ttl
|
|
|
6
6
|
use slatedb::object_store::memory::InMemory;
|
|
7
7
|
use slatedb::{Db, IsolationLevel};
|
|
8
8
|
|
|
9
|
-
use crate::errors::
|
|
9
|
+
use crate::errors::invalid_argument_error;
|
|
10
10
|
use crate::iterator::Iterator;
|
|
11
|
-
use crate::runtime::
|
|
11
|
+
use crate::runtime::block_on_result;
|
|
12
12
|
use crate::snapshot::Snapshot;
|
|
13
13
|
use crate::transaction::Transaction;
|
|
14
|
-
use crate::utils::get_optional;
|
|
14
|
+
use crate::utils::{get_optional, resolve_object_store};
|
|
15
15
|
use crate::write_batch::WriteBatch;
|
|
16
16
|
|
|
17
17
|
/// Ruby wrapper for SlateDB database.
|
|
@@ -32,18 +32,14 @@ impl Database {
|
|
|
32
32
|
/// # Returns
|
|
33
33
|
/// A new Database instance
|
|
34
34
|
pub fn open(path: String, url: Option<String>) -> Result<Self, Error> {
|
|
35
|
-
let db =
|
|
36
|
-
let object_store: Arc<dyn object_store::ObjectStore> = if let Some(ref
|
|
37
|
-
|
|
35
|
+
let db = block_on_result(async {
|
|
36
|
+
let object_store: Arc<dyn object_store::ObjectStore> = if let Some(ref url_str) = url {
|
|
37
|
+
resolve_object_store(url_str)?
|
|
38
38
|
} else {
|
|
39
|
-
// Use in-memory store for local testing
|
|
40
39
|
Arc::new(InMemory::new())
|
|
41
40
|
};
|
|
42
41
|
|
|
43
|
-
Db::builder(path, object_store)
|
|
44
|
-
.build()
|
|
45
|
-
.await
|
|
46
|
-
.map_err(map_error)
|
|
42
|
+
Db::builder(path, object_store).build().await
|
|
47
43
|
})?;
|
|
48
44
|
|
|
49
45
|
Ok(Self {
|
|
@@ -65,8 +61,8 @@ impl Database {
|
|
|
65
61
|
|
|
66
62
|
let opts = ReadOptions::default();
|
|
67
63
|
|
|
68
|
-
let result =
|
|
69
|
-
.
|
|
64
|
+
let result =
|
|
65
|
+
block_on_result(async { self.inner.get_with_options(key.as_bytes(), &opts).await })?;
|
|
70
66
|
|
|
71
67
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
|
72
68
|
}
|
|
@@ -105,8 +101,8 @@ impl Database {
|
|
|
105
101
|
opts.dirty = dirty;
|
|
106
102
|
}
|
|
107
103
|
|
|
108
|
-
let result =
|
|
109
|
-
.
|
|
104
|
+
let result =
|
|
105
|
+
block_on_result(async { self.inner.get_with_options(key.as_bytes(), &opts).await })?;
|
|
110
106
|
|
|
111
107
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
|
112
108
|
}
|
|
@@ -125,8 +121,8 @@ impl Database {
|
|
|
125
121
|
|
|
126
122
|
let opts = ReadOptions::default();
|
|
127
123
|
|
|
128
|
-
let result =
|
|
129
|
-
.
|
|
124
|
+
let result =
|
|
125
|
+
block_on_result(async { self.inner.get_with_options(key.as_bytes(), &opts).await })?;
|
|
130
126
|
|
|
131
127
|
Ok(result.map(|b| b.to_vec()))
|
|
132
128
|
}
|
|
@@ -147,12 +143,11 @@ impl Database {
|
|
|
147
143
|
await_durable: true,
|
|
148
144
|
};
|
|
149
145
|
|
|
150
|
-
|
|
146
|
+
block_on_result(async {
|
|
151
147
|
self.inner
|
|
152
148
|
.put_with_options(key.as_bytes(), value.as_bytes(), &put_opts, &write_opts)
|
|
153
149
|
.await
|
|
154
|
-
})
|
|
155
|
-
.map_err(map_error)?;
|
|
150
|
+
})?;
|
|
156
151
|
|
|
157
152
|
Ok(())
|
|
158
153
|
}
|
|
@@ -181,12 +176,11 @@ impl Database {
|
|
|
181
176
|
let await_durable = get_optional::<bool>(&kwargs, "await_durable")?.unwrap_or(true);
|
|
182
177
|
let write_opts = WriteOptions { await_durable };
|
|
183
178
|
|
|
184
|
-
|
|
179
|
+
block_on_result(async {
|
|
185
180
|
self.inner
|
|
186
181
|
.put_with_options(key.as_bytes(), value.as_bytes(), &put_opts, &write_opts)
|
|
187
182
|
.await
|
|
188
|
-
})
|
|
189
|
-
.map_err(map_error)?;
|
|
183
|
+
})?;
|
|
190
184
|
|
|
191
185
|
Ok(())
|
|
192
186
|
}
|
|
@@ -204,12 +198,11 @@ impl Database {
|
|
|
204
198
|
await_durable: true,
|
|
205
199
|
};
|
|
206
200
|
|
|
207
|
-
|
|
201
|
+
block_on_result(async {
|
|
208
202
|
self.inner
|
|
209
203
|
.delete_with_options(key.as_bytes(), &write_opts)
|
|
210
204
|
.await
|
|
211
|
-
})
|
|
212
|
-
.map_err(map_error)?;
|
|
205
|
+
})?;
|
|
213
206
|
|
|
214
207
|
Ok(())
|
|
215
208
|
}
|
|
@@ -227,12 +220,11 @@ impl Database {
|
|
|
227
220
|
let await_durable = get_optional::<bool>(&kwargs, "await_durable")?.unwrap_or(true);
|
|
228
221
|
let write_opts = WriteOptions { await_durable };
|
|
229
222
|
|
|
230
|
-
|
|
223
|
+
block_on_result(async {
|
|
231
224
|
self.inner
|
|
232
225
|
.delete_with_options(key.as_bytes(), &write_opts)
|
|
233
226
|
.await
|
|
234
|
-
})
|
|
235
|
-
.map_err(map_error)?;
|
|
227
|
+
})?;
|
|
236
228
|
|
|
237
229
|
Ok(())
|
|
238
230
|
}
|
|
@@ -255,12 +247,11 @@ impl Database {
|
|
|
255
247
|
let start_bytes = start.into_bytes();
|
|
256
248
|
let end_bytes = end_key.map(|e| e.into_bytes());
|
|
257
249
|
|
|
258
|
-
let iter =
|
|
259
|
-
|
|
250
|
+
let iter = block_on_result(async {
|
|
251
|
+
match end_bytes {
|
|
260
252
|
Some(end) => self.inner.scan_with_options(start_bytes..end, &opts).await,
|
|
261
253
|
None => self.inner.scan_with_options(start_bytes.., &opts).await,
|
|
262
|
-
}
|
|
263
|
-
range.map_err(map_error)
|
|
254
|
+
}
|
|
264
255
|
})?;
|
|
265
256
|
|
|
266
257
|
Ok(Iterator::new(iter))
|
|
@@ -324,12 +315,11 @@ impl Database {
|
|
|
324
315
|
let start_bytes = start.into_bytes();
|
|
325
316
|
let end_bytes = end_key.map(|e| e.into_bytes());
|
|
326
317
|
|
|
327
|
-
let iter =
|
|
328
|
-
|
|
318
|
+
let iter = block_on_result(async {
|
|
319
|
+
match end_bytes {
|
|
329
320
|
Some(end) => self.inner.scan_with_options(start_bytes..end, &opts).await,
|
|
330
321
|
None => self.inner.scan_with_options(start_bytes.., &opts).await,
|
|
331
|
-
}
|
|
332
|
-
range.map_err(map_error)
|
|
322
|
+
}
|
|
333
323
|
})?;
|
|
334
324
|
|
|
335
325
|
Ok(Iterator::new(iter))
|
|
@@ -341,9 +331,7 @@ impl Database {
|
|
|
341
331
|
/// * `batch` - The WriteBatch to write
|
|
342
332
|
pub fn write(&self, batch: &WriteBatch) -> Result<(), Error> {
|
|
343
333
|
let batch_inner = batch.take()?;
|
|
344
|
-
|
|
345
|
-
block_on(async { self.inner.write(batch_inner).await }).map_err(map_error)?;
|
|
346
|
-
|
|
334
|
+
block_on_result(async { self.inner.write(batch_inner).await })?;
|
|
347
335
|
Ok(())
|
|
348
336
|
}
|
|
349
337
|
|
|
@@ -358,12 +346,11 @@ impl Database {
|
|
|
358
346
|
|
|
359
347
|
let batch_inner = batch.take()?;
|
|
360
348
|
|
|
361
|
-
|
|
349
|
+
block_on_result(async {
|
|
362
350
|
self.inner
|
|
363
351
|
.write_with_options(batch_inner, &write_opts)
|
|
364
352
|
.await
|
|
365
|
-
})
|
|
366
|
-
.map_err(map_error)?;
|
|
353
|
+
})?;
|
|
367
354
|
|
|
368
355
|
Ok(())
|
|
369
356
|
}
|
|
@@ -389,8 +376,7 @@ impl Database {
|
|
|
389
376
|
}
|
|
390
377
|
};
|
|
391
378
|
|
|
392
|
-
let txn =
|
|
393
|
-
|
|
379
|
+
let txn = block_on_result(async { self.inner.begin(isolation_level).await })?;
|
|
394
380
|
Ok(Transaction::new(txn))
|
|
395
381
|
}
|
|
396
382
|
|
|
@@ -399,20 +385,19 @@ impl Database {
|
|
|
399
385
|
/// # Returns
|
|
400
386
|
/// A new Snapshot instance
|
|
401
387
|
pub fn snapshot(&self) -> Result<Snapshot, Error> {
|
|
402
|
-
let snap =
|
|
403
|
-
|
|
388
|
+
let snap = block_on_result(async { self.inner.snapshot().await })?;
|
|
404
389
|
Ok(Snapshot::new(snap))
|
|
405
390
|
}
|
|
406
391
|
|
|
407
392
|
/// Flush the database to ensure durability.
|
|
408
393
|
pub fn flush(&self) -> Result<(), Error> {
|
|
409
|
-
|
|
394
|
+
block_on_result(async { self.inner.flush().await })?;
|
|
410
395
|
Ok(())
|
|
411
396
|
}
|
|
412
397
|
|
|
413
398
|
/// Close the database.
|
|
414
399
|
pub fn close(&self) -> Result<(), Error> {
|
|
415
|
-
|
|
400
|
+
block_on_result(async { self.inner.close().await })?;
|
|
416
401
|
Ok(())
|
|
417
402
|
}
|
|
418
403
|
}
|
data/ext/slatedb/src/iterator.rs
CHANGED
|
@@ -5,11 +5,17 @@ use magnus::{method, Error, Ruby};
|
|
|
5
5
|
use slatedb::DbIterator;
|
|
6
6
|
use tokio::sync::Mutex;
|
|
7
7
|
|
|
8
|
+
use crate::errors::{internal_error, invalid_argument_error, map_error};
|
|
9
|
+
use crate::runtime::block_on;
|
|
10
|
+
|
|
8
11
|
/// Result type for raw byte key-value pairs.
|
|
9
12
|
type ByteKvResult = Result<Option<(Vec<u8>, Vec<u8>)>, Error>;
|
|
10
13
|
|
|
11
|
-
|
|
12
|
-
|
|
14
|
+
/// Internal error type for iterator operations (converted to Ruby errors after block_on).
|
|
15
|
+
enum IteratorError {
|
|
16
|
+
Closed,
|
|
17
|
+
Slate(slatedb::Error),
|
|
18
|
+
}
|
|
13
19
|
|
|
14
20
|
/// Ruby wrapper for SlateDB iterator.
|
|
15
21
|
///
|
|
@@ -36,14 +42,19 @@ impl Iterator {
|
|
|
36
42
|
|
|
37
43
|
let result = block_on(async {
|
|
38
44
|
let mut guard = inner.lock().await;
|
|
39
|
-
|
|
40
|
-
.
|
|
41
|
-
|
|
45
|
+
match guard.as_mut() {
|
|
46
|
+
Some(iter) => iter.next().await.map_err(IteratorError::Slate),
|
|
47
|
+
None => Err(IteratorError::Closed),
|
|
48
|
+
}
|
|
49
|
+
});
|
|
42
50
|
|
|
43
|
-
|
|
44
|
-
|
|
51
|
+
let kv = match result {
|
|
52
|
+
Ok(kv) => kv,
|
|
53
|
+
Err(IteratorError::Closed) => return Err(internal_error("iterator has been closed")),
|
|
54
|
+
Err(IteratorError::Slate(e)) => return Err(map_error(e)),
|
|
55
|
+
};
|
|
45
56
|
|
|
46
|
-
Ok(
|
|
57
|
+
Ok(kv.map(|kv| {
|
|
47
58
|
(
|
|
48
59
|
String::from_utf8_lossy(&kv.key).to_string(),
|
|
49
60
|
String::from_utf8_lossy(&kv.value).to_string(),
|
|
@@ -59,14 +70,19 @@ impl Iterator {
|
|
|
59
70
|
|
|
60
71
|
let result = block_on(async {
|
|
61
72
|
let mut guard = inner.lock().await;
|
|
62
|
-
|
|
63
|
-
.
|
|
64
|
-
|
|
73
|
+
match guard.as_mut() {
|
|
74
|
+
Some(iter) => iter.next().await.map_err(IteratorError::Slate),
|
|
75
|
+
None => Err(IteratorError::Closed),
|
|
76
|
+
}
|
|
77
|
+
});
|
|
65
78
|
|
|
66
|
-
|
|
67
|
-
|
|
79
|
+
let kv = match result {
|
|
80
|
+
Ok(kv) => kv,
|
|
81
|
+
Err(IteratorError::Closed) => return Err(internal_error("iterator has been closed")),
|
|
82
|
+
Err(IteratorError::Slate(e)) => return Err(map_error(e)),
|
|
83
|
+
};
|
|
68
84
|
|
|
69
|
-
Ok(
|
|
85
|
+
Ok(kv.map(|kv| (kv.key.to_vec(), kv.value.to_vec())))
|
|
70
86
|
}
|
|
71
87
|
|
|
72
88
|
/// Seek to a specific key position.
|
|
@@ -79,16 +95,19 @@ impl Iterator {
|
|
|
79
95
|
|
|
80
96
|
let inner = self.inner.clone();
|
|
81
97
|
|
|
82
|
-
block_on(async {
|
|
98
|
+
let result = block_on(async {
|
|
83
99
|
let mut guard = inner.lock().await;
|
|
84
|
-
|
|
85
|
-
.
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
})?;
|
|
100
|
+
match guard.as_mut() {
|
|
101
|
+
Some(iter) => iter.seek(key.as_bytes()).await.map_err(IteratorError::Slate),
|
|
102
|
+
None => Err(IteratorError::Closed),
|
|
103
|
+
}
|
|
104
|
+
});
|
|
90
105
|
|
|
91
|
-
|
|
106
|
+
match result {
|
|
107
|
+
Ok(()) => Ok(()),
|
|
108
|
+
Err(IteratorError::Closed) => Err(internal_error("iterator has been closed")),
|
|
109
|
+
Err(IteratorError::Slate(e)) => Err(map_error(e)),
|
|
110
|
+
}
|
|
92
111
|
}
|
|
93
112
|
|
|
94
113
|
/// Close the iterator and release resources.
|
data/ext/slatedb/src/reader.rs
CHANGED
|
@@ -5,10 +5,10 @@ use magnus::{function, method, Error, RHash, Ruby};
|
|
|
5
5
|
use slatedb::config::{DbReaderOptions, DurabilityLevel, ReadOptions, ScanOptions};
|
|
6
6
|
use slatedb::DbReader;
|
|
7
7
|
|
|
8
|
-
use crate::errors::
|
|
8
|
+
use crate::errors::invalid_argument_error;
|
|
9
9
|
use crate::iterator::Iterator;
|
|
10
|
-
use crate::runtime::
|
|
11
|
-
use crate::utils::get_optional;
|
|
10
|
+
use crate::runtime::block_on_result;
|
|
11
|
+
use crate::utils::{get_optional, resolve_object_store};
|
|
12
12
|
|
|
13
13
|
/// Ruby wrapper for SlateDB Reader.
|
|
14
14
|
///
|
|
@@ -50,9 +50,9 @@ impl Reader {
|
|
|
50
50
|
None
|
|
51
51
|
};
|
|
52
52
|
|
|
53
|
-
let reader =
|
|
53
|
+
let reader = block_on_result(async {
|
|
54
54
|
let object_store: Arc<dyn object_store::ObjectStore> = if let Some(ref url) = url {
|
|
55
|
-
|
|
55
|
+
resolve_object_store(url)?
|
|
56
56
|
} else {
|
|
57
57
|
Arc::new(object_store::memory::InMemory::new())
|
|
58
58
|
};
|
|
@@ -68,9 +68,7 @@ impl Reader {
|
|
|
68
68
|
options.max_memtable_bytes = max_bytes;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
DbReader::open(path, object_store, checkpoint_uuid, options)
|
|
72
|
-
.await
|
|
73
|
-
.map_err(map_error)
|
|
71
|
+
DbReader::open(path, object_store, checkpoint_uuid, options).await
|
|
74
72
|
})?;
|
|
75
73
|
|
|
76
74
|
Ok(Self {
|
|
@@ -84,8 +82,7 @@ impl Reader {
|
|
|
84
82
|
return Err(invalid_argument_error("key cannot be empty"));
|
|
85
83
|
}
|
|
86
84
|
|
|
87
|
-
let result =
|
|
88
|
-
|
|
85
|
+
let result = block_on_result(async { self.inner.get(key.as_bytes()).await })?;
|
|
89
86
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
|
90
87
|
}
|
|
91
88
|
|
|
@@ -114,9 +111,8 @@ impl Reader {
|
|
|
114
111
|
opts.dirty = dirty;
|
|
115
112
|
}
|
|
116
113
|
|
|
117
|
-
let result =
|
|
118
|
-
.
|
|
119
|
-
|
|
114
|
+
let result =
|
|
115
|
+
block_on_result(async { self.inner.get_with_options(key.as_bytes(), &opts).await })?;
|
|
120
116
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
|
121
117
|
}
|
|
122
118
|
|
|
@@ -126,8 +122,7 @@ impl Reader {
|
|
|
126
122
|
return Err(invalid_argument_error("key cannot be empty"));
|
|
127
123
|
}
|
|
128
124
|
|
|
129
|
-
let result =
|
|
130
|
-
|
|
125
|
+
let result = block_on_result(async { self.inner.get(key.as_bytes()).await })?;
|
|
131
126
|
Ok(result.map(|b| b.to_vec()))
|
|
132
127
|
}
|
|
133
128
|
|
|
@@ -140,12 +135,11 @@ impl Reader {
|
|
|
140
135
|
let start_bytes = start.into_bytes();
|
|
141
136
|
let end_bytes = end_key.map(|e| e.into_bytes());
|
|
142
137
|
|
|
143
|
-
let iter =
|
|
144
|
-
|
|
138
|
+
let iter = block_on_result(async {
|
|
139
|
+
match end_bytes {
|
|
145
140
|
Some(end) => self.inner.scan(start_bytes..end).await,
|
|
146
141
|
None => self.inner.scan(start_bytes..).await,
|
|
147
|
-
}
|
|
148
|
-
range.map_err(map_error)
|
|
142
|
+
}
|
|
149
143
|
})?;
|
|
150
144
|
|
|
151
145
|
Ok(Iterator::new(iter))
|
|
@@ -196,12 +190,11 @@ impl Reader {
|
|
|
196
190
|
let start_bytes = start.into_bytes();
|
|
197
191
|
let end_bytes = end_key.map(|e| e.into_bytes());
|
|
198
192
|
|
|
199
|
-
let iter =
|
|
200
|
-
|
|
193
|
+
let iter = block_on_result(async {
|
|
194
|
+
match end_bytes {
|
|
201
195
|
Some(end) => self.inner.scan_with_options(start_bytes..end, &opts).await,
|
|
202
196
|
None => self.inner.scan_with_options(start_bytes.., &opts).await,
|
|
203
|
-
}
|
|
204
|
-
range.map_err(map_error)
|
|
197
|
+
}
|
|
205
198
|
})?;
|
|
206
199
|
|
|
207
200
|
Ok(Iterator::new(iter))
|
|
@@ -209,7 +202,7 @@ impl Reader {
|
|
|
209
202
|
|
|
210
203
|
/// Close the reader.
|
|
211
204
|
pub fn close(&self) -> Result<(), Error> {
|
|
212
|
-
|
|
205
|
+
block_on_result(async { self.inner.close().await })?;
|
|
213
206
|
Ok(())
|
|
214
207
|
}
|
|
215
208
|
}
|
data/ext/slatedb/src/runtime.rs
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
use magnus::Error;
|
|
1
2
|
use once_cell::sync::OnceCell;
|
|
2
3
|
use rb_sys::rb_thread_call_without_gvl;
|
|
4
|
+
use slatedb::Error as SlateError;
|
|
3
5
|
use std::ffi::c_void;
|
|
4
6
|
use std::future::Future;
|
|
5
7
|
use tokio::runtime::Runtime;
|
|
6
8
|
|
|
9
|
+
use crate::errors::map_error;
|
|
10
|
+
|
|
7
11
|
static RUNTIME: OnceCell<Runtime> = OnceCell::new();
|
|
8
12
|
|
|
9
13
|
/// Get or initialize the shared Tokio runtime for all SlateDB operations.
|
|
@@ -22,20 +26,38 @@ fn get_runtime() -> &'static Runtime {
|
|
|
22
26
|
|
|
23
27
|
/// Execute a future on the runtime, releasing the Ruby GVL while waiting.
|
|
24
28
|
///
|
|
25
|
-
///
|
|
26
|
-
///
|
|
27
|
-
///
|
|
29
|
+
/// # GVL Safety
|
|
30
|
+
///
|
|
31
|
+
/// This function releases Ruby's Global VM Lock (GVL) while the future executes,
|
|
32
|
+
/// allowing other Ruby threads to run concurrently. This means:
|
|
33
|
+
///
|
|
34
|
+
/// - **Do NOT call Ruby APIs** inside the future (e.g., `Ruby::get()`, creating
|
|
35
|
+
/// Ruby exceptions, or calling magnus functions that require Ruby)
|
|
36
|
+
/// - **Do NOT use `map_error`** or other error converters inside the async block
|
|
37
|
+
/// - Return raw Rust types/errors from the future, then convert to Ruby types
|
|
38
|
+
/// after `block_on` returns (when the GVL is re-acquired)
|
|
39
|
+
///
|
|
40
|
+
/// For futures that return `Result<T, slatedb::Error>`, use [`block_on_result`]
|
|
41
|
+
/// which handles error conversion automatically.
|
|
28
42
|
pub fn block_on<F, T>(future: F) -> T
|
|
29
43
|
where
|
|
30
44
|
F: Future<Output = T>,
|
|
31
45
|
{
|
|
32
46
|
let rt = get_runtime();
|
|
33
|
-
|
|
34
|
-
// Use rb_thread_call_without_gvl to release the GVL while blocking
|
|
35
|
-
// This allows other Ruby threads to execute while we wait for I/O
|
|
36
47
|
without_gvl(|| rt.block_on(future))
|
|
37
48
|
}
|
|
38
49
|
|
|
50
|
+
/// Execute a future returning `Result<T, slatedb::Error>`, converting errors to Ruby.
|
|
51
|
+
///
|
|
52
|
+
/// This is a convenience wrapper around [`block_on`] that automatically converts
|
|
53
|
+
/// SlateDB errors to Ruby exceptions after the GVL is re-acquired.
|
|
54
|
+
pub fn block_on_result<F, T>(future: F) -> Result<T, Error>
|
|
55
|
+
where
|
|
56
|
+
F: Future<Output = Result<T, SlateError>>,
|
|
57
|
+
{
|
|
58
|
+
block_on(future).map_err(map_error)
|
|
59
|
+
}
|
|
60
|
+
|
|
39
61
|
/// Execute a closure without holding the Ruby GVL.
|
|
40
62
|
///
|
|
41
63
|
/// This releases the Global VM Lock, allowing other Ruby threads to run
|
data/ext/slatedb/src/snapshot.rs
CHANGED
|
@@ -6,9 +6,9 @@ use magnus::{method, Error, RHash, Ruby};
|
|
|
6
6
|
use slatedb::config::{DurabilityLevel, ReadOptions, ScanOptions};
|
|
7
7
|
use slatedb::DbSnapshot;
|
|
8
8
|
|
|
9
|
-
use crate::errors::{closed_error, invalid_argument_error
|
|
9
|
+
use crate::errors::{closed_error, invalid_argument_error};
|
|
10
10
|
use crate::iterator::Iterator;
|
|
11
|
-
use crate::runtime::
|
|
11
|
+
use crate::runtime::block_on_result;
|
|
12
12
|
use crate::utils::get_optional;
|
|
13
13
|
|
|
14
14
|
/// Ruby wrapper for SlateDB Snapshot.
|
|
@@ -39,8 +39,7 @@ impl Snapshot {
|
|
|
39
39
|
.as_ref()
|
|
40
40
|
.ok_or_else(|| closed_error("snapshot is closed"))?;
|
|
41
41
|
|
|
42
|
-
let result =
|
|
43
|
-
|
|
42
|
+
let result = block_on_result(async { snapshot.get(key.as_bytes()).await })?;
|
|
44
43
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
|
45
44
|
}
|
|
46
45
|
|
|
@@ -74,9 +73,8 @@ impl Snapshot {
|
|
|
74
73
|
.as_ref()
|
|
75
74
|
.ok_or_else(|| closed_error("snapshot is closed"))?;
|
|
76
75
|
|
|
77
|
-
let result =
|
|
78
|
-
.
|
|
79
|
-
|
|
76
|
+
let result =
|
|
77
|
+
block_on_result(async { snapshot.get_with_options(key.as_bytes(), &opts).await })?;
|
|
80
78
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
|
81
79
|
}
|
|
82
80
|
|
|
@@ -94,12 +92,11 @@ impl Snapshot {
|
|
|
94
92
|
let start_bytes = start.into_bytes();
|
|
95
93
|
let end_bytes = end_key.map(|e| e.into_bytes());
|
|
96
94
|
|
|
97
|
-
let iter =
|
|
98
|
-
|
|
95
|
+
let iter = block_on_result(async {
|
|
96
|
+
match end_bytes {
|
|
99
97
|
Some(end) => snapshot.scan(start_bytes..end).await,
|
|
100
98
|
None => snapshot.scan(start_bytes..).await,
|
|
101
|
-
}
|
|
102
|
-
range.map_err(map_error)
|
|
99
|
+
}
|
|
103
100
|
})?;
|
|
104
101
|
|
|
105
102
|
Ok(Iterator::new(iter))
|
|
@@ -155,12 +152,11 @@ impl Snapshot {
|
|
|
155
152
|
let start_bytes = start.into_bytes();
|
|
156
153
|
let end_bytes = end_key.map(|e| e.into_bytes());
|
|
157
154
|
|
|
158
|
-
let iter =
|
|
159
|
-
|
|
155
|
+
let iter = block_on_result(async {
|
|
156
|
+
match end_bytes {
|
|
160
157
|
Some(end) => snapshot.scan_with_options(start_bytes..end, &opts).await,
|
|
161
158
|
None => snapshot.scan_with_options(start_bytes.., &opts).await,
|
|
162
|
-
}
|
|
163
|
-
range.map_err(map_error)
|
|
159
|
+
}
|
|
164
160
|
})?;
|
|
165
161
|
|
|
166
162
|
Ok(Iterator::new(iter))
|
|
@@ -7,7 +7,7 @@ use slatedb::DBTransaction;
|
|
|
7
7
|
|
|
8
8
|
use crate::errors::{closed_error, invalid_argument_error, map_error};
|
|
9
9
|
use crate::iterator::Iterator;
|
|
10
|
-
use crate::runtime::
|
|
10
|
+
use crate::runtime::block_on_result;
|
|
11
11
|
use crate::utils::get_optional;
|
|
12
12
|
|
|
13
13
|
/// Ruby wrapper for SlateDB Transaction.
|
|
@@ -38,8 +38,7 @@ impl Transaction {
|
|
|
38
38
|
.as_ref()
|
|
39
39
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
40
40
|
|
|
41
|
-
let result =
|
|
42
|
-
|
|
41
|
+
let result = block_on_result(async { txn.get(key.as_bytes()).await })?;
|
|
43
42
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
|
44
43
|
}
|
|
45
44
|
|
|
@@ -73,9 +72,8 @@ impl Transaction {
|
|
|
73
72
|
.as_ref()
|
|
74
73
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
75
74
|
|
|
76
|
-
let result =
|
|
77
|
-
.
|
|
78
|
-
|
|
75
|
+
let result =
|
|
76
|
+
block_on_result(async { txn.get_with_options(key.as_bytes(), &opts).await })?;
|
|
79
77
|
Ok(result.map(|b| String::from_utf8_lossy(&b).to_string()))
|
|
80
78
|
}
|
|
81
79
|
|
|
@@ -151,12 +149,11 @@ impl Transaction {
|
|
|
151
149
|
let start_bytes = start.into_bytes();
|
|
152
150
|
let end_bytes = end_key.map(|e| e.into_bytes());
|
|
153
151
|
|
|
154
|
-
let iter =
|
|
155
|
-
|
|
152
|
+
let iter = block_on_result(async {
|
|
153
|
+
match end_bytes {
|
|
156
154
|
Some(end) => txn.scan(start_bytes..end).await,
|
|
157
155
|
None => txn.scan(start_bytes..).await,
|
|
158
|
-
}
|
|
159
|
-
range.map_err(map_error)
|
|
156
|
+
}
|
|
160
157
|
})?;
|
|
161
158
|
|
|
162
159
|
Ok(Iterator::new(iter))
|
|
@@ -212,12 +209,11 @@ impl Transaction {
|
|
|
212
209
|
let start_bytes = start.into_bytes();
|
|
213
210
|
let end_bytes = end_key.map(|e| e.into_bytes());
|
|
214
211
|
|
|
215
|
-
let iter =
|
|
216
|
-
|
|
212
|
+
let iter = block_on_result(async {
|
|
213
|
+
match end_bytes {
|
|
217
214
|
Some(end) => txn.scan_with_options(start_bytes..end, &opts).await,
|
|
218
215
|
None => txn.scan_with_options(start_bytes.., &opts).await,
|
|
219
|
-
}
|
|
220
|
-
range.map_err(map_error)
|
|
216
|
+
}
|
|
221
217
|
})?;
|
|
222
218
|
|
|
223
219
|
Ok(Iterator::new(iter))
|
|
@@ -231,8 +227,7 @@ impl Transaction {
|
|
|
231
227
|
.take()
|
|
232
228
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
233
229
|
|
|
234
|
-
|
|
235
|
-
|
|
230
|
+
block_on_result(async { txn.commit().await })?;
|
|
236
231
|
Ok(())
|
|
237
232
|
}
|
|
238
233
|
|
|
@@ -247,8 +242,7 @@ impl Transaction {
|
|
|
247
242
|
.take()
|
|
248
243
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
249
244
|
|
|
250
|
-
|
|
251
|
-
|
|
245
|
+
block_on_result(async { txn.commit_with_options(&write_opts).await })?;
|
|
252
246
|
Ok(())
|
|
253
247
|
}
|
|
254
248
|
|
data/ext/slatedb/src/utils.rs
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
|
+
use std::sync::Arc;
|
|
2
|
+
|
|
1
3
|
use magnus::value::ReprValue;
|
|
2
4
|
use magnus::{Error, RHash, Ruby, TryConvert};
|
|
5
|
+
use object_store::aws::AmazonS3Builder;
|
|
6
|
+
use object_store::ObjectStoreScheme;
|
|
7
|
+
use slatedb::{Db, Error as SlateError};
|
|
8
|
+
use url::Url;
|
|
3
9
|
|
|
4
10
|
/// Helper to extract an optional value from an RHash
|
|
5
11
|
pub fn get_optional<T: TryConvert>(hash: &RHash, key: &str) -> Result<Option<T>, Error> {
|
|
@@ -16,3 +22,37 @@ pub fn get_optional<T: TryConvert>(hash: &RHash, key: &str) -> Result<Option<T>,
|
|
|
16
22
|
None => Ok(None),
|
|
17
23
|
}
|
|
18
24
|
}
|
|
25
|
+
|
|
26
|
+
/// Convert an object_store error to a SlateDB error
|
|
27
|
+
fn to_slate_error(e: object_store::Error) -> SlateError {
|
|
28
|
+
SlateError::unavailable(e.to_string())
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/// Resolve an object store URL to an ObjectStore instance.
|
|
32
|
+
///
|
|
33
|
+
/// This function handles S3 URLs specially to ensure environment variables
|
|
34
|
+
/// like AWS_ACCESS_KEY_ID are properly recognized (the default object_store
|
|
35
|
+
/// registry only recognizes lowercase variants like aws_access_key_id).
|
|
36
|
+
pub fn resolve_object_store(url: &str) -> Result<Arc<dyn object_store::ObjectStore>, SlateError> {
|
|
37
|
+
let parsed_url: Url = url
|
|
38
|
+
.try_into()
|
|
39
|
+
.map_err(|e: url::ParseError| SlateError::invalid(format!("invalid URL: {}", e)))?;
|
|
40
|
+
|
|
41
|
+
let (scheme, _path) =
|
|
42
|
+
ObjectStoreScheme::parse(&parsed_url).map_err(|e| to_slate_error(e.into()))?;
|
|
43
|
+
|
|
44
|
+
match scheme {
|
|
45
|
+
ObjectStoreScheme::AmazonS3 => {
|
|
46
|
+
// Use from_env() to properly handle uppercase AWS_* environment variables
|
|
47
|
+
let store = AmazonS3Builder::from_env()
|
|
48
|
+
.with_url(url)
|
|
49
|
+
.build()
|
|
50
|
+
.map_err(to_slate_error)?;
|
|
51
|
+
Ok(Arc::new(store))
|
|
52
|
+
}
|
|
53
|
+
_ => {
|
|
54
|
+
// Fall back to slatedb's default resolver for other schemes
|
|
55
|
+
Db::resolve_object_store(url)
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
data/lib/slatedb/database.rb
CHANGED
data/lib/slatedb/version.rb
CHANGED