slatedb 0.3.1 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +65 -7
- data/ext/slatedb/Cargo.toml +1 -1
- data/ext/slatedb/src/admin.rs +5 -0
- data/ext/slatedb/src/database.rs +4 -3
- data/ext/slatedb/src/reader.rs +35 -15
- data/ext/slatedb/src/snapshot.rs +5 -3
- data/ext/slatedb/src/transaction.rs +6 -3
- data/ext/slatedb/src/utils.rs +46 -4
- data/lib/slatedb/database.rb +13 -1
- data/lib/slatedb/reader.rb +29 -9
- data/lib/slatedb/snapshot.rb +15 -7
- data/lib/slatedb/transaction.rb +15 -7
- data/lib/slatedb/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5ea58f5cc108fef4066004dce3db7641d979589c6afa56023845b4e568198630
|
|
4
|
+
data.tar.gz: c5e52bded1c840e6d51ba420b673c5f03e3f9450dd78476dc129a45cab02b11b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 715e35df65d25a1487f648718ec72e593fed5e22289f47c2017aa8eac24ccda5d5df00151ec618b1d732550be2f8b6ac5d8136b9358d18c197ae11b5e437e8a9
|
|
7
|
+
data.tar.gz: 92780ae93284906f92a4e4550d885c281af32f7cd1f91642e6ca9ebbe1f114d574d20a17be3f0605b5caaaafe64e27e70f12a96b5fcc6255fd8f0e210f0fa9b1
|
data/README.md
CHANGED
|
@@ -8,10 +8,6 @@ Ruby bindings for [SlateDB](https://slatedb.io), a cloud-native embedded key-val
|
|
|
8
8
|
|
|
9
9
|
These bindings are still in early development, and while SlateDB itself is used in Production, these bindings have yet to be. Contributions are welcome!
|
|
10
10
|
|
|
11
|
-
### TODO
|
|
12
|
-
|
|
13
|
-
- [ ] Cross-compile native extensions
|
|
14
|
-
|
|
15
11
|
## Installation
|
|
16
12
|
|
|
17
13
|
Add this line to your application's Gemfile:
|
|
@@ -32,8 +28,7 @@ Or install it yourself as:
|
|
|
32
28
|
gem install slatedb
|
|
33
29
|
```
|
|
34
30
|
|
|
35
|
-
|
|
36
|
-
> This gem currently requires a working Rust toolchain to install until the dependencies are cross-compiled.
|
|
31
|
+
Precompiled native gems are published for common Linux and Darwin platforms. Building from source still requires a Rust toolchain.
|
|
37
32
|
|
|
38
33
|
## Usage
|
|
39
34
|
|
|
@@ -268,6 +263,27 @@ db.transaction do |txn|
|
|
|
268
263
|
end
|
|
269
264
|
```
|
|
270
265
|
|
|
266
|
+
You can narrow a prefix scan to a sub-range using the `from:` (inclusive) and
|
|
267
|
+
`to:` (exclusive) options (SlateDB >= 0.14.0). Both are key *suffixes* that are
|
|
268
|
+
appended to the prefix, so they let you paginate or resume within a prefix
|
|
269
|
+
without building full-key ranges by hand. The scan never escapes the prefix.
|
|
270
|
+
|
|
271
|
+
```ruby
|
|
272
|
+
# Keys "user:100" (inclusive) up to "user:200" (exclusive)
|
|
273
|
+
db.scan_prefix("user:", from: "100", to: "200").each do |key, value|
|
|
274
|
+
puts "#{key}: #{value}"
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# Only a lower bound: from "user:500" to the end of the prefix
|
|
278
|
+
db.scan_prefix("user:", from: "500")
|
|
279
|
+
|
|
280
|
+
# Only an upper bound: from the start of the prefix up to (but not including) "user:100"
|
|
281
|
+
db.scan_prefix("user:", to: "100")
|
|
282
|
+
|
|
283
|
+
# Sub-ranges compose with order: and work on transactions, snapshots, and readers
|
|
284
|
+
db.scan_prefix("user:", from: "100", to: "200", order: :desc)
|
|
285
|
+
```
|
|
286
|
+
|
|
271
287
|
### Merge Operations
|
|
272
288
|
|
|
273
289
|
Merge operations allow you to combine values without reading them first, useful for counters, append-only logs, and similar patterns:
|
|
@@ -487,13 +503,24 @@ SlateDb::Reader.open("/tmp/mydb", url: "s3://bucket/path") do |reader|
|
|
|
487
503
|
end
|
|
488
504
|
end
|
|
489
505
|
|
|
490
|
-
# Open at a specific checkpoint
|
|
506
|
+
# Open at a specific checkpoint (pins the reader to that checkpoint's state)
|
|
491
507
|
SlateDb::Reader.open("/tmp/mydb",
|
|
492
508
|
url: "s3://bucket/path",
|
|
493
509
|
checkpoint_id: "uuid-here") do |reader|
|
|
494
510
|
reader.get("key")
|
|
495
511
|
end
|
|
496
512
|
|
|
513
|
+
# Follow the latest state without managing a checkpoint (SlateDB >= 0.15.0).
|
|
514
|
+
# This performs no object-store writes but offers no protection from garbage
|
|
515
|
+
# collection, so it suits read-only or mirrored databases. By default (neither
|
|
516
|
+
# checkpoint_id nor follow_latest set) the reader creates and periodically
|
|
517
|
+
# refreshes its own checkpoint instead.
|
|
518
|
+
SlateDb::Reader.open("/tmp/mydb",
|
|
519
|
+
url: "s3://bucket/path",
|
|
520
|
+
follow_latest: true) do |reader|
|
|
521
|
+
reader.get("key")
|
|
522
|
+
end
|
|
523
|
+
|
|
497
524
|
# Enable the reader's on-disk cache and cap its open file handles
|
|
498
525
|
# (max_open_file_handles, added in SlateDB 0.13.0, only takes effect when
|
|
499
526
|
# cache_root is set, since that is what enables the cached object store).
|
|
@@ -604,6 +631,37 @@ Exception hierarchy:
|
|
|
604
631
|
- Ruby 3.3+
|
|
605
632
|
- Rust toolchain (for building from source)
|
|
606
633
|
|
|
634
|
+
## Releasing
|
|
635
|
+
|
|
636
|
+
Releases publish a generic `ruby` platform gem and **precompiled native gems**
|
|
637
|
+
for six platforms (`x86_64-linux`, `aarch64-linux`, `x86_64-linux-musl`,
|
|
638
|
+
`aarch64-linux-musl`, `arm64-darwin`, `x86_64-darwin`) to
|
|
639
|
+
[rubygems.org](https://rubygems.org/gems/slatedb). The generic gem keeps
|
|
640
|
+
RubyGems' latest-version metadata aligned; supported platforms should still
|
|
641
|
+
resolve to their matching precompiled native gem.
|
|
642
|
+
|
|
643
|
+
### Cutting a release
|
|
644
|
+
|
|
645
|
+
The gem version comes from `lib/slatedb/version.rb`, not from the tag. Use the
|
|
646
|
+
mise helper to bump, commit, tag, and push in one step (must be on `main` with a
|
|
647
|
+
clean working tree):
|
|
648
|
+
|
|
649
|
+
```bash
|
|
650
|
+
mise run release:cut 0.4.3
|
|
651
|
+
# or: mise run release:cut 0.4.3 --no-push
|
|
652
|
+
```
|
|
653
|
+
|
|
654
|
+
That updates `SlateDb::VERSION`, commits `Release v0.4.3`, creates tag `v0.4.3`,
|
|
655
|
+
and pushes the commit + tag (unless `--no-push`). The tag triggers the release
|
|
656
|
+
pipeline.
|
|
657
|
+
|
|
658
|
+
The `release:verify-tag` step fails the build if the tag and
|
|
659
|
+
`SlateDb::VERSION` disagree, so the tag (`v0.4.3`) must match `version.rb`
|
|
660
|
+
(`0.4.3`).
|
|
661
|
+
|
|
662
|
+
To exercise packaging without publishing, trigger a manual build on the
|
|
663
|
+
`slatedb-rb-release` pipeline or set `DRY_RUN=true`.
|
|
664
|
+
|
|
607
665
|
## Development
|
|
608
666
|
|
|
609
667
|
After checking out the repo, run:
|
data/ext/slatedb/Cargo.toml
CHANGED
|
@@ -11,7 +11,7 @@ name = "slatedb"
|
|
|
11
11
|
crate-type = ["cdylib"]
|
|
12
12
|
|
|
13
13
|
[dependencies]
|
|
14
|
-
slatedb = "0.
|
|
14
|
+
slatedb = "0.15.0"
|
|
15
15
|
magnus = { version = "0.8.2", features = ["rb-sys"] }
|
|
16
16
|
rb-sys = { version = "0.9.128", features = ["stable-api-compiled-fallback"] }
|
|
17
17
|
tokio = { version = "1.52.3", features = ["rt-multi-thread", "sync"] }
|
data/ext/slatedb/src/admin.rs
CHANGED
|
@@ -230,6 +230,7 @@ impl Admin {
|
|
|
230
230
|
Some(GarbageCollectorDirectoryOptions {
|
|
231
231
|
interval: default_opts.as_ref().and_then(|o| o.interval),
|
|
232
232
|
min_age: std::time::Duration::from_millis(ms),
|
|
233
|
+
dry_run: default_opts.as_ref().map(|o| o.dry_run).unwrap_or(false),
|
|
233
234
|
})
|
|
234
235
|
} else {
|
|
235
236
|
default_opts
|
|
@@ -243,6 +244,7 @@ impl Admin {
|
|
|
243
244
|
default_opts.manifest_options,
|
|
244
245
|
),
|
|
245
246
|
wal_options: make_dir_opts(wal_min_age, min_age, default_opts.wal_options),
|
|
247
|
+
wal_fence_options: default_opts.wal_fence_options,
|
|
246
248
|
compacted_options: make_dir_opts(
|
|
247
249
|
compacted_min_age,
|
|
248
250
|
min_age,
|
|
@@ -250,6 +252,9 @@ impl Admin {
|
|
|
250
252
|
),
|
|
251
253
|
compactions_options: default_opts.compactions_options,
|
|
252
254
|
detach_options: default_opts.detach_options,
|
|
255
|
+
metric_level: default_opts.metric_level,
|
|
256
|
+
boundary_files_enabled: default_opts.boundary_files_enabled,
|
|
257
|
+
object_store_max_retries: default_opts.object_store_max_retries,
|
|
253
258
|
}
|
|
254
259
|
};
|
|
255
260
|
|
data/ext/slatedb/src/database.rs
CHANGED
|
@@ -16,7 +16,7 @@ use crate::metrics::Metrics;
|
|
|
16
16
|
use crate::runtime::block_on_result;
|
|
17
17
|
use crate::snapshot::Snapshot;
|
|
18
18
|
use crate::transaction::Transaction;
|
|
19
|
-
use crate::utils::{get_optional, resolve_object_store};
|
|
19
|
+
use crate::utils::{get_optional, prefix_subrange_from_kwargs, resolve_object_store};
|
|
20
20
|
use crate::write_batch::WriteBatch;
|
|
21
21
|
|
|
22
22
|
/// Ruby wrapper for SlateDB database.
|
|
@@ -467,7 +467,7 @@ impl Database {
|
|
|
467
467
|
let opts = ScanOptions::default();
|
|
468
468
|
let iter = block_on_result(async {
|
|
469
469
|
self.inner
|
|
470
|
-
.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
470
|
+
.scan_prefix_with_options(prefix.as_bytes(), .., &opts)
|
|
471
471
|
.await
|
|
472
472
|
})?;
|
|
473
473
|
|
|
@@ -534,9 +534,10 @@ impl Database {
|
|
|
534
534
|
};
|
|
535
535
|
}
|
|
536
536
|
|
|
537
|
+
let subrange = prefix_subrange_from_kwargs(&kwargs)?;
|
|
537
538
|
let iter = block_on_result(async {
|
|
538
539
|
self.inner
|
|
539
|
-
.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
540
|
+
.scan_prefix_with_options(prefix.as_bytes(), subrange, &opts)
|
|
540
541
|
.await
|
|
541
542
|
})?;
|
|
542
543
|
|
data/ext/slatedb/src/reader.rs
CHANGED
|
@@ -4,12 +4,13 @@ use magnus::prelude::*;
|
|
|
4
4
|
use magnus::{function, method, Error, RHash, Ruby};
|
|
5
5
|
use slatedb::config::{DbReaderOptions, DurabilityLevel, ReadOptions, ScanOptions};
|
|
6
6
|
use slatedb::DbReader;
|
|
7
|
+
use slatedb::DbReaderMode;
|
|
7
8
|
use slatedb::IterationOrder;
|
|
8
9
|
|
|
9
10
|
use crate::errors::invalid_argument_error;
|
|
10
11
|
use crate::iterator::Iterator;
|
|
11
12
|
use crate::runtime::block_on_result;
|
|
12
|
-
use crate::utils::{get_optional, resolve_object_store};
|
|
13
|
+
use crate::utils::{get_optional, prefix_subrange_from_kwargs, resolve_object_store};
|
|
13
14
|
|
|
14
15
|
/// Ruby wrapper for SlateDB Reader.
|
|
15
16
|
///
|
|
@@ -26,11 +27,20 @@ impl Reader {
|
|
|
26
27
|
/// # Arguments
|
|
27
28
|
/// * `path` - The path identifier for the database
|
|
28
29
|
/// * `url` - Optional object store URL
|
|
29
|
-
/// * `checkpoint_id` - Optional checkpoint UUID to read at
|
|
30
|
+
/// * `checkpoint_id` - Optional checkpoint UUID to read at. When set, the reader
|
|
31
|
+
/// is pinned to that checkpoint ([`DbReaderMode::Checkpoint`]).
|
|
30
32
|
/// * `kwargs` - Additional options (manifest_poll_interval, checkpoint_lifetime,
|
|
31
|
-
/// max_memtable_bytes, skip_wal_replay, cache_root, max_open_file_handles
|
|
33
|
+
/// max_memtable_bytes, skip_wal_replay, cache_root, max_open_file_handles,
|
|
34
|
+
/// follow_latest).
|
|
32
35
|
/// The local disk cache (and therefore `max_open_file_handles`) is only active
|
|
33
36
|
/// when `cache_root` is set.
|
|
37
|
+
///
|
|
38
|
+
/// When neither `checkpoint_id` nor `follow_latest` is given the reader defaults
|
|
39
|
+
/// to [`DbReaderMode::ManagedCheckpoint`], creating and periodically refreshing
|
|
40
|
+
/// its own checkpoint so garbage collection cannot delete objects out from under
|
|
41
|
+
/// it. Setting `follow_latest: true` selects [`DbReaderMode::FollowLatest`], which
|
|
42
|
+
/// tails the latest manifest without writing any checkpoint — useful for read-only
|
|
43
|
+
/// or mirrored databases, at the cost of no protection from garbage collection.
|
|
34
44
|
pub fn open(
|
|
35
45
|
path: String,
|
|
36
46
|
url: Option<String>,
|
|
@@ -46,16 +56,24 @@ impl Reader {
|
|
|
46
56
|
let skip_wal_replay = get_optional::<bool>(&kwargs, "skip_wal_replay")?;
|
|
47
57
|
let max_open_file_handles = get_optional::<usize>(&kwargs, "max_open_file_handles")?;
|
|
48
58
|
let cache_root = get_optional::<String>(&kwargs, "cache_root")?;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
59
|
+
let follow_latest = get_optional::<bool>(&kwargs, "follow_latest")?.unwrap_or(false);
|
|
60
|
+
|
|
61
|
+
// Resolve the reader mode from checkpoint_id / follow_latest.
|
|
62
|
+
let mode = match checkpoint_id {
|
|
63
|
+
Some(id_str) => {
|
|
64
|
+
if follow_latest {
|
|
65
|
+
return Err(invalid_argument_error(
|
|
66
|
+
"checkpoint_id and follow_latest are mutually exclusive",
|
|
67
|
+
));
|
|
68
|
+
}
|
|
69
|
+
let uuid = uuid::Uuid::parse_str(&id_str).map_err(|e| {
|
|
54
70
|
invalid_argument_error(&format!("invalid checkpoint_id: {}", e))
|
|
55
|
-
})
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
71
|
+
})?;
|
|
72
|
+
DbReaderMode::Checkpoint(uuid)
|
|
73
|
+
}
|
|
74
|
+
None if follow_latest => DbReaderMode::FollowLatest,
|
|
75
|
+
None => DbReaderMode::ManagedCheckpoint,
|
|
76
|
+
};
|
|
59
77
|
|
|
60
78
|
let reader = block_on_result(async {
|
|
61
79
|
let object_store: Arc<dyn slatedb::object_store::ObjectStore> =
|
|
@@ -85,7 +103,7 @@ impl Reader {
|
|
|
85
103
|
if let Some(max_handles) = max_open_file_handles {
|
|
86
104
|
options.object_store_cache_options.max_open_file_handles = max_handles;
|
|
87
105
|
}
|
|
88
|
-
DbReader::open(path, object_store,
|
|
106
|
+
DbReader::open(path, object_store, mode, options).await
|
|
89
107
|
})?;
|
|
90
108
|
|
|
91
109
|
Ok(Self {
|
|
@@ -239,7 +257,8 @@ impl Reader {
|
|
|
239
257
|
return Err(invalid_argument_error("prefix cannot be empty"));
|
|
240
258
|
}
|
|
241
259
|
|
|
242
|
-
let iter =
|
|
260
|
+
let iter =
|
|
261
|
+
block_on_result(async { self.inner.scan_prefix(prefix.as_bytes(), ..).await })?;
|
|
243
262
|
|
|
244
263
|
Ok(Iterator::new(iter))
|
|
245
264
|
}
|
|
@@ -297,9 +316,10 @@ impl Reader {
|
|
|
297
316
|
};
|
|
298
317
|
}
|
|
299
318
|
|
|
319
|
+
let subrange = prefix_subrange_from_kwargs(&kwargs)?;
|
|
300
320
|
let iter = block_on_result(async {
|
|
301
321
|
self.inner
|
|
302
|
-
.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
322
|
+
.scan_prefix_with_options(prefix.as_bytes(), subrange, &opts)
|
|
303
323
|
.await
|
|
304
324
|
})?;
|
|
305
325
|
|
data/ext/slatedb/src/snapshot.rs
CHANGED
|
@@ -10,7 +10,7 @@ use slatedb::IterationOrder;
|
|
|
10
10
|
use crate::errors::{closed_error, invalid_argument_error};
|
|
11
11
|
use crate::iterator::Iterator;
|
|
12
12
|
use crate::runtime::block_on_result;
|
|
13
|
-
use crate::utils::get_optional;
|
|
13
|
+
use crate::utils::{get_optional, prefix_subrange_from_kwargs};
|
|
14
14
|
|
|
15
15
|
/// Ruby wrapper for SlateDB Snapshot.
|
|
16
16
|
///
|
|
@@ -190,7 +190,7 @@ impl Snapshot {
|
|
|
190
190
|
.as_ref()
|
|
191
191
|
.ok_or_else(|| closed_error("snapshot is closed"))?;
|
|
192
192
|
|
|
193
|
-
let iter = block_on_result(async { snapshot.scan_prefix(prefix.as_bytes()).await })?;
|
|
193
|
+
let iter = block_on_result(async { snapshot.scan_prefix(prefix.as_bytes(), ..).await })?;
|
|
194
194
|
|
|
195
195
|
Ok(Iterator::new(iter))
|
|
196
196
|
}
|
|
@@ -248,6 +248,8 @@ impl Snapshot {
|
|
|
248
248
|
};
|
|
249
249
|
}
|
|
250
250
|
|
|
251
|
+
let subrange = prefix_subrange_from_kwargs(&kwargs)?;
|
|
252
|
+
|
|
251
253
|
let guard = self.inner.borrow();
|
|
252
254
|
let snapshot = guard
|
|
253
255
|
.as_ref()
|
|
@@ -255,7 +257,7 @@ impl Snapshot {
|
|
|
255
257
|
|
|
256
258
|
let iter = block_on_result(async {
|
|
257
259
|
snapshot
|
|
258
|
-
.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
260
|
+
.scan_prefix_with_options(prefix.as_bytes(), subrange, &opts)
|
|
259
261
|
.await
|
|
260
262
|
})?;
|
|
261
263
|
|
|
@@ -11,7 +11,7 @@ use slatedb::IterationOrder;
|
|
|
11
11
|
use crate::errors::{closed_error, invalid_argument_error, map_error};
|
|
12
12
|
use crate::iterator::Iterator;
|
|
13
13
|
use crate::runtime::block_on_result;
|
|
14
|
-
use crate::utils::get_optional;
|
|
14
|
+
use crate::utils::{get_optional, prefix_subrange_from_kwargs};
|
|
15
15
|
|
|
16
16
|
/// Ruby wrapper for SlateDB Transaction.
|
|
17
17
|
///
|
|
@@ -295,7 +295,7 @@ impl Transaction {
|
|
|
295
295
|
.as_ref()
|
|
296
296
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
297
297
|
|
|
298
|
-
let iter = block_on_result(async { txn.scan_prefix(prefix.as_bytes()).await })?;
|
|
298
|
+
let iter = block_on_result(async { txn.scan_prefix(prefix.as_bytes(), ..).await })?;
|
|
299
299
|
|
|
300
300
|
Ok(Iterator::new(iter))
|
|
301
301
|
}
|
|
@@ -353,13 +353,16 @@ impl Transaction {
|
|
|
353
353
|
};
|
|
354
354
|
}
|
|
355
355
|
|
|
356
|
+
let subrange = prefix_subrange_from_kwargs(&kwargs)?;
|
|
357
|
+
|
|
356
358
|
let guard = self.inner.borrow();
|
|
357
359
|
let txn = guard
|
|
358
360
|
.as_ref()
|
|
359
361
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
360
362
|
|
|
361
363
|
let iter = block_on_result(async {
|
|
362
|
-
txn.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
364
|
+
txn.scan_prefix_with_options(prefix.as_bytes(), subrange, &opts)
|
|
365
|
+
.await
|
|
363
366
|
})?;
|
|
364
367
|
|
|
365
368
|
Ok(Iterator::new(iter))
|
data/ext/slatedb/src/utils.rs
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
|
+
use std::ops::Bound;
|
|
1
2
|
use std::sync::Arc;
|
|
2
3
|
|
|
3
4
|
use magnus::value::ReprValue;
|
|
4
5
|
use magnus::{Error, RHash, Ruby, TryConvert};
|
|
5
6
|
use slatedb::object_store::aws::AmazonS3Builder;
|
|
6
|
-
use slatedb::object_store::
|
|
7
|
-
use slatedb::{
|
|
7
|
+
use slatedb::object_store::prefix::PrefixStore;
|
|
8
|
+
use slatedb::object_store::{
|
|
9
|
+
parse_url_opts, Error as ObjectStoreError, ObjectStore, ObjectStoreScheme,
|
|
10
|
+
};
|
|
11
|
+
use slatedb::Error as SlateError;
|
|
8
12
|
use url::Url;
|
|
9
13
|
|
|
10
14
|
/// Helper to extract an optional value from an RHash
|
|
@@ -23,6 +27,30 @@ pub fn get_optional<T: TryConvert>(hash: &RHash, key: &str) -> Result<Option<T>,
|
|
|
23
27
|
}
|
|
24
28
|
}
|
|
25
29
|
|
|
30
|
+
/// A key sub-range for a prefix scan, expressed as owned byte bounds. This
|
|
31
|
+
/// implements `slatedb::bytes_range::ByteRangeBounds`, so it can be passed
|
|
32
|
+
/// straight to `scan_prefix`/`scan_prefix_with_options`.
|
|
33
|
+
pub type PrefixSubrange = (Bound<Vec<u8>>, Bound<Vec<u8>>);
|
|
34
|
+
|
|
35
|
+
/// Build a suffix sub-range (relative to a scan prefix) from optional
|
|
36
|
+
/// `subrange_from` / `subrange_to` keyword arguments.
|
|
37
|
+
///
|
|
38
|
+
/// The bounds are key *suffixes* appended to the prefix: `subrange_from` is an
|
|
39
|
+
/// inclusive lower bound and `subrange_to` is an exclusive upper bound. A
|
|
40
|
+
/// missing bound is unbounded, so an empty hash yields a full-prefix scan
|
|
41
|
+
/// (equivalent to `..`).
|
|
42
|
+
pub fn prefix_subrange_from_kwargs(hash: &RHash) -> Result<PrefixSubrange, Error> {
|
|
43
|
+
let start = match get_optional::<String>(hash, "subrange_from")? {
|
|
44
|
+
Some(s) => Bound::Included(s.into_bytes()),
|
|
45
|
+
None => Bound::Unbounded,
|
|
46
|
+
};
|
|
47
|
+
let end = match get_optional::<String>(hash, "subrange_to")? {
|
|
48
|
+
Some(s) => Bound::Excluded(s.into_bytes()),
|
|
49
|
+
None => Bound::Unbounded,
|
|
50
|
+
};
|
|
51
|
+
Ok((start, end))
|
|
52
|
+
}
|
|
53
|
+
|
|
26
54
|
/// Convert an object_store error to a SlateDB error
|
|
27
55
|
fn to_slate_error(e: ObjectStoreError) -> SlateError {
|
|
28
56
|
SlateError::unavailable(e.to_string())
|
|
@@ -52,8 +80,22 @@ pub fn resolve_object_store(url: &str) -> Result<Arc<dyn ObjectStore>, SlateErro
|
|
|
52
80
|
Ok(Arc::new(store))
|
|
53
81
|
}
|
|
54
82
|
_ => {
|
|
55
|
-
//
|
|
56
|
-
|
|
83
|
+
// SlateDB 0.14 changed `Db::resolve_object_store` to reject any URL
|
|
84
|
+
// that carries a path component (returning `InvalidObjectStorePath`),
|
|
85
|
+
// whereas 0.13 transparently wrapped it in a `PrefixStore`. Preserve
|
|
86
|
+
// the old, more forgiving behavior here so callers can keep passing a
|
|
87
|
+
// full location such as "file:///data/mydb" or "gs://bucket/prefix".
|
|
88
|
+
//
|
|
89
|
+
// Env keys are lowercased because `parse_url_opts` only recognizes
|
|
90
|
+
// lower-case option keys.
|
|
91
|
+
let env_vars = std::env::vars().map(|(k, v)| (k.to_ascii_lowercase(), v));
|
|
92
|
+
let (store, path) = parse_url_opts(&parsed_url, env_vars).map_err(to_slate_error)?;
|
|
93
|
+
let store: Arc<dyn ObjectStore> = Arc::from(store);
|
|
94
|
+
if path.as_ref().is_empty() {
|
|
95
|
+
Ok(store)
|
|
96
|
+
} else {
|
|
97
|
+
Ok(Arc::new(PrefixStore::new(store, path)))
|
|
98
|
+
}
|
|
57
99
|
}
|
|
58
100
|
}
|
|
59
101
|
}
|
data/lib/slatedb/database.rb
CHANGED
|
@@ -246,6 +246,11 @@ module SlateDb
|
|
|
246
246
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
247
247
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
248
248
|
# @param order [Symbol, String, nil] Iteration order (:asc/:ascending or :desc/:descending)
|
|
249
|
+
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
250
|
+
# prefix, to start scanning from (e.g. prefix "user:" with from "100"
|
|
251
|
+
# starts at "user:100"). Defaults to the start of the prefix.
|
|
252
|
+
# @param to [String, nil] Exclusive upper bound suffix, appended to the
|
|
253
|
+
# prefix, to stop scanning at. Defaults to the end of the prefix.
|
|
249
254
|
# @return [Iterator] An iterator over key-value pairs
|
|
250
255
|
#
|
|
251
256
|
# @example Scan all user keys
|
|
@@ -253,8 +258,13 @@ module SlateDb
|
|
|
253
258
|
# puts "#{key}: #{value}"
|
|
254
259
|
# end
|
|
255
260
|
#
|
|
261
|
+
# @example Scan a sub-range within a prefix
|
|
262
|
+
# # keys "user:100" (inclusive) up to "user:200" (exclusive)
|
|
263
|
+
# db.scan_prefix("user:", from: "100", to: "200")
|
|
264
|
+
#
|
|
256
265
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
257
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, order: nil,
|
|
266
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, order: nil,
|
|
267
|
+
from: nil, to: nil, &)
|
|
258
268
|
opts = scan_options(
|
|
259
269
|
durability_filter: durability_filter,
|
|
260
270
|
dirty: dirty,
|
|
@@ -263,6 +273,8 @@ module SlateDb
|
|
|
263
273
|
max_fetch_tasks: max_fetch_tasks,
|
|
264
274
|
order: order
|
|
265
275
|
)
|
|
276
|
+
opts[:subrange_from] = from if from
|
|
277
|
+
opts[:subrange_to] = to if to
|
|
266
278
|
|
|
267
279
|
iter = if opts.empty?
|
|
268
280
|
_scan_prefix(prefix)
|
data/lib/slatedb/reader.rb
CHANGED
|
@@ -7,7 +7,14 @@ module SlateDb
|
|
|
7
7
|
#
|
|
8
8
|
# @param path [String] The path identifier for the database
|
|
9
9
|
# @param url [String, nil] Optional object store URL
|
|
10
|
-
# @param checkpoint_id [String, nil] Optional checkpoint UUID to read at
|
|
10
|
+
# @param checkpoint_id [String, nil] Optional checkpoint UUID to read at. When
|
|
11
|
+
# given, the reader is pinned to that checkpoint and does not follow new writes.
|
|
12
|
+
# @param follow_latest [Boolean, nil] When true, the reader tails the latest
|
|
13
|
+
# manifest without creating or maintaining its own checkpoint. This performs no
|
|
14
|
+
# object-store writes but provides no protection from garbage collection, so it
|
|
15
|
+
# is best suited to read-only or mirrored databases. Mutually exclusive with
|
|
16
|
+
# +checkpoint_id+. When neither is set (the default), the reader manages its own
|
|
17
|
+
# checkpoint and refreshes it periodically. (Requires SlateDB >= 0.15.0)
|
|
11
18
|
# @param manifest_poll_interval [Integer, nil] Poll interval in milliseconds
|
|
12
19
|
# @param checkpoint_lifetime [Integer, nil] Checkpoint lifetime in milliseconds
|
|
13
20
|
# @param max_memtable_bytes [Integer, nil] Maximum memtable size in bytes
|
|
@@ -35,16 +42,20 @@ module SlateDb
|
|
|
35
42
|
# @example Open at a specific checkpoint
|
|
36
43
|
# reader = SlateDb::Reader.open("/tmp/mydb", checkpoint_id: "uuid-here")
|
|
37
44
|
#
|
|
45
|
+
# @example Follow the latest state without managing a checkpoint
|
|
46
|
+
# reader = SlateDb::Reader.open("/tmp/mydb", follow_latest: true)
|
|
47
|
+
#
|
|
38
48
|
# @example Enable the on-disk cache and cap its open file handles
|
|
39
49
|
# reader = SlateDb::Reader.open("/tmp/mydb",
|
|
40
50
|
# cache_root: "/var/cache/slatedb",
|
|
41
51
|
# max_open_file_handles: 256)
|
|
42
52
|
#
|
|
43
|
-
def open(path, url: nil, checkpoint_id: nil,
|
|
53
|
+
def open(path, url: nil, checkpoint_id: nil, follow_latest: nil,
|
|
44
54
|
manifest_poll_interval: nil, checkpoint_lifetime: nil,
|
|
45
55
|
max_memtable_bytes: nil, cache_root: nil, max_open_file_handles: nil,
|
|
46
56
|
merge_operator: nil)
|
|
47
57
|
opts = {}
|
|
58
|
+
opts[:follow_latest] = follow_latest unless follow_latest.nil?
|
|
48
59
|
opts[:manifest_poll_interval] = manifest_poll_interval if manifest_poll_interval
|
|
49
60
|
opts[:checkpoint_lifetime] = checkpoint_lifetime if checkpoint_lifetime
|
|
50
61
|
opts[:max_memtable_bytes] = max_memtable_bytes if max_memtable_bytes
|
|
@@ -127,16 +138,25 @@ module SlateDb
|
|
|
127
138
|
# @param read_ahead_bytes [Integer, nil] Number of bytes to read ahead
|
|
128
139
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
129
140
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
141
|
+
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
142
|
+
# prefix, to start scanning from (e.g. prefix "user:" with from "100"
|
|
143
|
+
# starts at "user:100"). Defaults to the start of the prefix.
|
|
144
|
+
# @param to [String, nil] Exclusive upper bound suffix, appended to the
|
|
145
|
+
# prefix, to stop scanning at. Defaults to the end of the prefix.
|
|
130
146
|
# @return [Iterator] An iterator over key-value pairs
|
|
131
147
|
#
|
|
132
148
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
133
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
134
|
-
|
|
135
|
-
opts
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
149
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
150
|
+
from: nil, to: nil, &)
|
|
151
|
+
opts = {
|
|
152
|
+
durability_filter: durability_filter&.to_s,
|
|
153
|
+
dirty: dirty,
|
|
154
|
+
read_ahead_bytes: read_ahead_bytes,
|
|
155
|
+
cache_blocks: cache_blocks,
|
|
156
|
+
max_fetch_tasks: max_fetch_tasks,
|
|
157
|
+
subrange_from: from,
|
|
158
|
+
subrange_to: to
|
|
159
|
+
}.compact
|
|
140
160
|
|
|
141
161
|
iter = if opts.empty?
|
|
142
162
|
_scan_prefix(prefix)
|
data/lib/slatedb/snapshot.rb
CHANGED
|
@@ -59,16 +59,24 @@ module SlateDb
|
|
|
59
59
|
# @param read_ahead_bytes [Integer, nil] Number of bytes to read ahead
|
|
60
60
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
61
61
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
62
|
+
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
63
|
+
# prefix, to start scanning from. Defaults to the start of the prefix.
|
|
64
|
+
# @param to [String, nil] Exclusive upper bound suffix, appended to the
|
|
65
|
+
# prefix, to stop scanning at. Defaults to the end of the prefix.
|
|
62
66
|
# @return [Iterator] An iterator over key-value pairs
|
|
63
67
|
#
|
|
64
68
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
65
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
66
|
-
|
|
67
|
-
opts
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
70
|
+
from: nil, to: nil, &)
|
|
71
|
+
opts = {
|
|
72
|
+
durability_filter: durability_filter&.to_s,
|
|
73
|
+
dirty: dirty,
|
|
74
|
+
read_ahead_bytes: read_ahead_bytes,
|
|
75
|
+
cache_blocks: cache_blocks,
|
|
76
|
+
max_fetch_tasks: max_fetch_tasks,
|
|
77
|
+
subrange_from: from,
|
|
78
|
+
subrange_to: to
|
|
79
|
+
}.compact
|
|
72
80
|
|
|
73
81
|
iter = if opts.empty?
|
|
74
82
|
_scan_prefix(prefix)
|
data/lib/slatedb/transaction.rb
CHANGED
|
@@ -98,16 +98,24 @@ module SlateDb
|
|
|
98
98
|
# @param read_ahead_bytes [Integer, nil] Number of bytes to read ahead
|
|
99
99
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
100
100
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
101
|
+
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
102
|
+
# prefix, to start scanning from. Defaults to the start of the prefix.
|
|
103
|
+
# @param to [String, nil] Exclusive upper bound suffix, appended to the
|
|
104
|
+
# prefix, to stop scanning at. Defaults to the end of the prefix.
|
|
101
105
|
# @return [Iterator] An iterator over key-value pairs
|
|
102
106
|
#
|
|
103
107
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
104
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
105
|
-
|
|
106
|
-
opts
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
108
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
109
|
+
from: nil, to: nil, &)
|
|
110
|
+
opts = {
|
|
111
|
+
durability_filter: durability_filter&.to_s,
|
|
112
|
+
dirty: dirty,
|
|
113
|
+
read_ahead_bytes: read_ahead_bytes,
|
|
114
|
+
cache_blocks: cache_blocks,
|
|
115
|
+
max_fetch_tasks: max_fetch_tasks,
|
|
116
|
+
subrange_from: from,
|
|
117
|
+
subrange_to: to
|
|
118
|
+
}.compact
|
|
111
119
|
|
|
112
120
|
iter = if opts.empty?
|
|
113
121
|
_scan_prefix(prefix)
|
data/lib/slatedb/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: slatedb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- SlateDB Contributors
|
|
@@ -137,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
137
137
|
- !ruby/object:Gem::Version
|
|
138
138
|
version: '0'
|
|
139
139
|
requirements: []
|
|
140
|
-
rubygems_version: 4.0.
|
|
140
|
+
rubygems_version: 4.0.16
|
|
141
141
|
specification_version: 4
|
|
142
142
|
summary: Ruby bindings for SlateDB
|
|
143
143
|
test_files: []
|