slatedb 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +96 -7
- data/ext/slatedb/Cargo.toml +1 -1
- data/ext/slatedb/src/admin.rs +26 -0
- data/ext/slatedb/src/database.rs +122 -43
- data/ext/slatedb/src/reader.rs +35 -15
- data/ext/slatedb/src/snapshot.rs +5 -3
- data/ext/slatedb/src/transaction.rs +27 -11
- data/ext/slatedb/src/utils.rs +46 -4
- data/ext/slatedb/src/write_batch.rs +2 -2
- data/lib/slatedb/admin.rb +28 -0
- data/lib/slatedb/database.rb +71 -1
- data/lib/slatedb/reader.rb +29 -9
- data/lib/slatedb/snapshot.rb +15 -7
- data/lib/slatedb/transaction.rb +15 -7
- data/lib/slatedb/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: dd1077018f7a8738545fc60b626a0e51d3240ae4141d9616787bf642489a22bc
|
|
4
|
+
data.tar.gz: 7aea1d1d0f9e1e3ee349d742246b7f89353ab8af39a6aca12d3167d7064d9023
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2fcbdc5f116120c4d06bb857407047505a5ca5c526a9c8209cb06411635adc506b9a634024db9152c23f5b98f46d9600d2cf21d4826cf5a947b335cb72449eb2
|
|
7
|
+
data.tar.gz: e40b8fde694ca637e5ab80d529e161eefe8901cf478a768ebbc2ddd35b9cc16f42e8858feb4cd149333e7e3db6f4ee977770485390c56303ba8dba3fd88803ff
|
data/README.md
CHANGED
|
@@ -8,10 +8,6 @@ Ruby bindings for [SlateDB](https://slatedb.io), a cloud-native embedded key-val
|
|
|
8
8
|
|
|
9
9
|
These bindings are still in early development, and while SlateDB itself is used in Production, these bindings have yet to be. Contributions are welcome!
|
|
10
10
|
|
|
11
|
-
### TODO
|
|
12
|
-
|
|
13
|
-
- [ ] Cross-compile native extensions
|
|
14
|
-
|
|
15
11
|
## Installation
|
|
16
12
|
|
|
17
13
|
Add this line to your application's Gemfile:
|
|
@@ -32,8 +28,7 @@ Or install it yourself as:
|
|
|
32
28
|
gem install slatedb
|
|
33
29
|
```
|
|
34
30
|
|
|
35
|
-
|
|
36
|
-
> This gem currently requires a working Rust toolchain to install until the dependencies are cross-compiled.
|
|
31
|
+
Precompiled native gems are published for common Linux and Darwin platforms. Building from source still requires a Rust toolchain.
|
|
37
32
|
|
|
38
33
|
## Usage
|
|
39
34
|
|
|
@@ -268,6 +263,27 @@ db.transaction do |txn|
|
|
|
268
263
|
end
|
|
269
264
|
```
|
|
270
265
|
|
|
266
|
+
You can narrow a prefix scan to a sub-range using the `from:` (inclusive) and
|
|
267
|
+
`to:` (exclusive) options (SlateDB >= 0.14.0). Both are key *suffixes* that are
|
|
268
|
+
appended to the prefix, so they let you paginate or resume within a prefix
|
|
269
|
+
without building full-key ranges by hand. The scan never escapes the prefix.
|
|
270
|
+
|
|
271
|
+
```ruby
|
|
272
|
+
# Keys "user:100" (inclusive) up to "user:200" (exclusive)
|
|
273
|
+
db.scan_prefix("user:", from: "100", to: "200").each do |key, value|
|
|
274
|
+
puts "#{key}: #{value}"
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# Only a lower bound: from "user:500" to the end of the prefix
|
|
278
|
+
db.scan_prefix("user:", from: "500")
|
|
279
|
+
|
|
280
|
+
# Only an upper bound: from the start of the prefix up to (but not including) "user:100"
|
|
281
|
+
db.scan_prefix("user:", to: "100")
|
|
282
|
+
|
|
283
|
+
# Sub-ranges compose with order: and work on transactions, snapshots, and readers
|
|
284
|
+
db.scan_prefix("user:", from: "100", to: "200", order: :desc)
|
|
285
|
+
```
|
|
286
|
+
|
|
271
287
|
### Merge Operations
|
|
272
288
|
|
|
273
289
|
Merge operations allow you to combine values without reading them first, useful for counters, append-only logs, and similar patterns:
|
|
@@ -487,13 +503,24 @@ SlateDb::Reader.open("/tmp/mydb", url: "s3://bucket/path") do |reader|
|
|
|
487
503
|
end
|
|
488
504
|
end
|
|
489
505
|
|
|
490
|
-
# Open at a specific checkpoint
|
|
506
|
+
# Open at a specific checkpoint (pins the reader to that checkpoint's state)
|
|
491
507
|
SlateDb::Reader.open("/tmp/mydb",
|
|
492
508
|
url: "s3://bucket/path",
|
|
493
509
|
checkpoint_id: "uuid-here") do |reader|
|
|
494
510
|
reader.get("key")
|
|
495
511
|
end
|
|
496
512
|
|
|
513
|
+
# Follow the latest state without managing a checkpoint (SlateDB >= 0.15.0).
|
|
514
|
+
# This performs no object-store writes but offers no protection from garbage
|
|
515
|
+
# collection, so it suits read-only or mirrored databases. By default (neither
|
|
516
|
+
# checkpoint_id nor follow_latest set) the reader creates and periodically
|
|
517
|
+
# refreshes its own checkpoint instead.
|
|
518
|
+
SlateDb::Reader.open("/tmp/mydb",
|
|
519
|
+
url: "s3://bucket/path",
|
|
520
|
+
follow_latest: true) do |reader|
|
|
521
|
+
reader.get("key")
|
|
522
|
+
end
|
|
523
|
+
|
|
497
524
|
# Enable the reader's on-disk cache and cap its open file handles
|
|
498
525
|
# (max_open_file_handles, added in SlateDB 0.13.0, only takes effect when
|
|
499
526
|
# cache_root is set, since that is what enables the cached object store).
|
|
@@ -534,8 +561,19 @@ admin.run_gc(min_age: 3600_000) # Set min age for all directorie
|
|
|
534
561
|
admin.run_gc(manifest_min_age: 86400_000) # Custom age for manifest (1 day)
|
|
535
562
|
admin.run_gc(wal_min_age: 60_000) # Custom age for WAL (1 minute)
|
|
536
563
|
admin.run_gc(compacted_min_age: 60_000) # Custom age for compacted (1 minute)
|
|
564
|
+
|
|
565
|
+
# Delete a database (SlateDB >= 0.16.0)
|
|
566
|
+
would_delete = admin.delete_db # Dry run: returns the object paths
|
|
567
|
+
# that *would* be deleted, deleting nothing
|
|
568
|
+
deleted = admin.delete_db(confirm: true) # Actually delete; returns the removed paths
|
|
537
569
|
```
|
|
538
570
|
|
|
571
|
+
`delete_db` is idempotent, and a confirmed delete of a directory that is not a
|
|
572
|
+
SlateDB database (no manifest and no in-progress deletion marker) is refused, so
|
|
573
|
+
a mistyped path cannot wipe an unrelated directory. For a cloned database, any
|
|
574
|
+
checkpoints it pinned in parent databases are stripped before its own objects
|
|
575
|
+
are removed.
|
|
576
|
+
|
|
539
577
|
### Flushing
|
|
540
578
|
|
|
541
579
|
Ensure all writes are persisted:
|
|
@@ -545,6 +583,26 @@ db.put("key", "value")
|
|
|
545
583
|
db.flush
|
|
546
584
|
```
|
|
547
585
|
|
|
586
|
+
By default, `flush` flushes the write-ahead log (WAL) if enabled, otherwise the
|
|
587
|
+
memtable. You can choose explicitly with `flush_type:` (SlateDB >= 0.16.0):
|
|
588
|
+
|
|
589
|
+
```ruby
|
|
590
|
+
db.flush(flush_type: :wal) # Flush the write-ahead log
|
|
591
|
+
db.flush(flush_type: :memtable) # Freeze and flush memtable entries to object storage
|
|
592
|
+
```
|
|
593
|
+
|
|
594
|
+
### Closing
|
|
595
|
+
|
|
596
|
+
Closing flushes the active memtable first by default, so durable writes are not
|
|
597
|
+
lost. You can control the final flush when closing (SlateDB >= 0.16.0):
|
|
598
|
+
|
|
599
|
+
```ruby
|
|
600
|
+
db.close # Flush the memtable, then close (default)
|
|
601
|
+
db.close(flush_type: :wal) # Flush the WAL before closing
|
|
602
|
+
db.close(flush: false) # Close without a final flush; writes that are not
|
|
603
|
+
# yet durable may be lost
|
|
604
|
+
```
|
|
605
|
+
|
|
548
606
|
## Thread Safety
|
|
549
607
|
|
|
550
608
|
**SlateDB is fully thread-safe and optimized for concurrent access.**
|
|
@@ -604,6 +662,37 @@ Exception hierarchy:
|
|
|
604
662
|
- Ruby 3.3+
|
|
605
663
|
- Rust toolchain (for building from source)
|
|
606
664
|
|
|
665
|
+
## Releasing
|
|
666
|
+
|
|
667
|
+
Releases publish a generic `ruby` platform gem and **precompiled native gems**
|
|
668
|
+
for six platforms (`x86_64-linux`, `aarch64-linux`, `x86_64-linux-musl`,
|
|
669
|
+
`aarch64-linux-musl`, `arm64-darwin`, `x86_64-darwin`) to
|
|
670
|
+
[rubygems.org](https://rubygems.org/gems/slatedb). The generic gem keeps
|
|
671
|
+
RubyGems' latest-version metadata aligned; supported platforms should still
|
|
672
|
+
resolve to their matching precompiled native gem.
|
|
673
|
+
|
|
674
|
+
### Cutting a release
|
|
675
|
+
|
|
676
|
+
The gem version comes from `lib/slatedb/version.rb`, not from the tag. Use the
|
|
677
|
+
mise helper to bump, commit, tag, and push in one step (must be on `main` with a
|
|
678
|
+
clean working tree):
|
|
679
|
+
|
|
680
|
+
```bash
|
|
681
|
+
mise run release:cut 0.4.3
|
|
682
|
+
# or: mise run release:cut 0.4.3 --no-push
|
|
683
|
+
```
|
|
684
|
+
|
|
685
|
+
That updates `SlateDb::VERSION`, commits `Release v0.4.3`, creates tag `v0.4.3`,
|
|
686
|
+
and pushes the commit + tag (unless `--no-push`). The tag triggers the release
|
|
687
|
+
pipeline.
|
|
688
|
+
|
|
689
|
+
The `release:verify-tag` step fails the build if the tag and
|
|
690
|
+
`SlateDb::VERSION` disagree, so the tag (`v0.4.3`) must match `version.rb`
|
|
691
|
+
(`0.4.3`).
|
|
692
|
+
|
|
693
|
+
To exercise packaging without publishing, trigger a manual build on the
|
|
694
|
+
`slatedb-rb-release` pipeline or set `DRY_RUN=true`.
|
|
695
|
+
|
|
607
696
|
## Development
|
|
608
697
|
|
|
609
698
|
After checking out the repo, run:
|
data/ext/slatedb/Cargo.toml
CHANGED
|
@@ -11,7 +11,7 @@ name = "slatedb"
|
|
|
11
11
|
crate-type = ["cdylib"]
|
|
12
12
|
|
|
13
13
|
[dependencies]
|
|
14
|
-
slatedb = "0.
|
|
14
|
+
slatedb = "0.16.0"
|
|
15
15
|
magnus = { version = "0.8.2", features = ["rb-sys"] }
|
|
16
16
|
rb-sys = { version = "0.9.128", features = ["stable-api-compiled-fallback"] }
|
|
17
17
|
tokio = { version = "1.52.3", features = ["rt-multi-thread", "sync"] }
|
data/ext/slatedb/src/admin.rs
CHANGED
|
@@ -230,6 +230,7 @@ impl Admin {
|
|
|
230
230
|
Some(GarbageCollectorDirectoryOptions {
|
|
231
231
|
interval: default_opts.as_ref().and_then(|o| o.interval),
|
|
232
232
|
min_age: std::time::Duration::from_millis(ms),
|
|
233
|
+
dry_run: default_opts.as_ref().map(|o| o.dry_run).unwrap_or(false),
|
|
233
234
|
})
|
|
234
235
|
} else {
|
|
235
236
|
default_opts
|
|
@@ -243,6 +244,7 @@ impl Admin {
|
|
|
243
244
|
default_opts.manifest_options,
|
|
244
245
|
),
|
|
245
246
|
wal_options: make_dir_opts(wal_min_age, min_age, default_opts.wal_options),
|
|
247
|
+
wal_fence_options: default_opts.wal_fence_options,
|
|
246
248
|
compacted_options: make_dir_opts(
|
|
247
249
|
compacted_min_age,
|
|
248
250
|
min_age,
|
|
@@ -250,6 +252,9 @@ impl Admin {
|
|
|
250
252
|
),
|
|
251
253
|
compactions_options: default_opts.compactions_options,
|
|
252
254
|
detach_options: default_opts.detach_options,
|
|
255
|
+
metric_level: default_opts.metric_level,
|
|
256
|
+
boundary_files_enabled: default_opts.boundary_files_enabled,
|
|
257
|
+
object_store_max_retries: default_opts.object_store_max_retries,
|
|
253
258
|
}
|
|
254
259
|
};
|
|
255
260
|
|
|
@@ -260,6 +265,26 @@ impl Admin {
|
|
|
260
265
|
|
|
261
266
|
Ok(())
|
|
262
267
|
}
|
|
268
|
+
|
|
269
|
+
/// Delete a database, stripping any checkpoints it pinned in parent
|
|
270
|
+
/// databases (for clones) before removing its own objects (SlateDB >= 0.16.0).
|
|
271
|
+
///
|
|
272
|
+
/// # Arguments
|
|
273
|
+
/// * `confirm` - When `false` (the default), performs a dry run and returns
|
|
274
|
+
/// the list of object paths that *would* be deleted without touching
|
|
275
|
+
/// anything. When `true`, actually deletes and returns the object paths
|
|
276
|
+
/// that were removed.
|
|
277
|
+
///
|
|
278
|
+
/// A `confirm` delete of a directory with neither a manifest nor a leftover
|
|
279
|
+
/// `.deleting` marker is refused, so an incorrect path cannot wipe an
|
|
280
|
+
/// unrelated directory. The operation is idempotent.
|
|
281
|
+
///
|
|
282
|
+
/// # Returns
|
|
283
|
+
/// An array of object paths (deleted, or that would be deleted for a dry run).
|
|
284
|
+
pub fn delete_db(&self, confirm: bool) -> Result<Vec<String>, Error> {
|
|
285
|
+
let deleted = block_on_result(async { self.inner.delete_db(confirm).await })?;
|
|
286
|
+
Ok(deleted)
|
|
287
|
+
}
|
|
263
288
|
}
|
|
264
289
|
|
|
265
290
|
/// Define the Admin class on the SlateDb module.
|
|
@@ -277,6 +302,7 @@ pub fn define_admin_class(ruby: &Ruby, module: &magnus::RModule) -> Result<(), E
|
|
|
277
302
|
class.define_method("_refresh_checkpoint", method!(Admin::refresh_checkpoint, 2))?;
|
|
278
303
|
class.define_method("_delete_checkpoint", method!(Admin::delete_checkpoint, 1))?;
|
|
279
304
|
class.define_method("_run_gc", method!(Admin::run_gc, 1))?;
|
|
305
|
+
class.define_method("_delete_db", method!(Admin::delete_db, 1))?;
|
|
280
306
|
|
|
281
307
|
Ok(())
|
|
282
308
|
}
|
data/ext/slatedb/src/database.rs
CHANGED
|
@@ -4,7 +4,8 @@ use std::sync::{Arc, Mutex};
|
|
|
4
4
|
use magnus::prelude::*;
|
|
5
5
|
use magnus::{function, method, Error, RHash, Ruby};
|
|
6
6
|
use slatedb::config::{
|
|
7
|
-
|
|
7
|
+
CloseOptions, DurabilityLevel, FlushOptions, FlushType, MergeOptions, PutOptions, ReadOptions,
|
|
8
|
+
ScanOptions, Ttl, WriteOptions,
|
|
8
9
|
};
|
|
9
10
|
use slatedb::object_store::memory::InMemory;
|
|
10
11
|
use slatedb::{Db, IsolationLevel, IterationOrder, KeyValue};
|
|
@@ -16,7 +17,7 @@ use crate::metrics::Metrics;
|
|
|
16
17
|
use crate::runtime::block_on_result;
|
|
17
18
|
use crate::snapshot::Snapshot;
|
|
18
19
|
use crate::transaction::Transaction;
|
|
19
|
-
use crate::utils::{get_optional, resolve_object_store};
|
|
20
|
+
use crate::utils::{get_optional, prefix_subrange_from_kwargs, resolve_object_store};
|
|
20
21
|
use crate::write_batch::WriteBatch;
|
|
21
22
|
|
|
22
23
|
/// Ruby wrapper for SlateDB database.
|
|
@@ -79,13 +80,32 @@ impl Database {
|
|
|
79
80
|
}
|
|
80
81
|
|
|
81
82
|
fn write_options_from_kwargs(kwargs: &RHash) -> Result<WriteOptions, Error> {
|
|
82
|
-
let await_durable = get_optional::<bool>(kwargs, "await_durable")?.unwrap_or(true);
|
|
83
83
|
let seqnum = get_optional::<u64>(kwargs, "seqnum")?.unwrap_or(0);
|
|
84
84
|
|
|
85
|
-
Ok(WriteOptions {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
85
|
+
Ok(WriteOptions { seqnum })
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/// Parse the `await_durable` keyword argument (defaults to `true`).
|
|
89
|
+
///
|
|
90
|
+
/// As of SlateDB 0.16.0, `await_durable` is no longer a `WriteOptions`
|
|
91
|
+
/// field. Write methods return immediately after updating the in-memory
|
|
92
|
+
/// WAL/MemTable and hand back a `WriteHandle`; blocking until the write is
|
|
93
|
+
/// durable in object storage is done by awaiting that handle. This helper
|
|
94
|
+
/// preserves the previous ergonomic `await_durable:` behavior for callers.
|
|
95
|
+
fn await_durable_from_kwargs(kwargs: &RHash) -> Result<bool, Error> {
|
|
96
|
+
Ok(get_optional::<bool>(kwargs, "await_durable")?.unwrap_or(true))
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/// Parse a `flush_type` name into a SlateDB `FlushType` (SlateDB >= 0.16.0).
|
|
100
|
+
fn parse_flush_type(name: &str) -> Result<FlushType, Error> {
|
|
101
|
+
match name {
|
|
102
|
+
"wal" => Ok(FlushType::Wal),
|
|
103
|
+
"memtable" | "mem_table" => Ok(FlushType::MemTable),
|
|
104
|
+
other => Err(invalid_argument_error(&format!(
|
|
105
|
+
"invalid flush_type: {} (expected 'wal' or 'memtable')",
|
|
106
|
+
other
|
|
107
|
+
))),
|
|
108
|
+
}
|
|
89
109
|
}
|
|
90
110
|
|
|
91
111
|
/// Open a database at the given path.
|
|
@@ -249,16 +269,14 @@ impl Database {
|
|
|
249
269
|
}
|
|
250
270
|
|
|
251
271
|
let put_opts = PutOptions { ttl: Ttl::Default };
|
|
252
|
-
|
|
253
|
-
let write_opts = WriteOptions {
|
|
254
|
-
await_durable: true,
|
|
255
|
-
seqnum: 0,
|
|
256
|
-
};
|
|
272
|
+
let write_opts = WriteOptions::default();
|
|
257
273
|
|
|
258
274
|
block_on_result(async {
|
|
259
|
-
self
|
|
275
|
+
let handle = self
|
|
276
|
+
.inner
|
|
260
277
|
.put_with_options(key.as_bytes(), value.as_bytes(), &put_opts, &write_opts)
|
|
261
|
-
.await
|
|
278
|
+
.await?;
|
|
279
|
+
handle.await_durable().await
|
|
262
280
|
})?;
|
|
263
281
|
self.increment_metric("db.put.count");
|
|
264
282
|
|
|
@@ -280,18 +298,23 @@ impl Database {
|
|
|
280
298
|
let ttl = get_optional::<u64>(&kwargs, "ttl")?;
|
|
281
299
|
let put_opts = PutOptions {
|
|
282
300
|
ttl: match ttl {
|
|
283
|
-
Some(ms) => Ttl::
|
|
301
|
+
Some(ms) => Ttl::ExpireAfterMillis(ms),
|
|
284
302
|
None => Ttl::Default,
|
|
285
303
|
},
|
|
286
304
|
};
|
|
287
305
|
|
|
288
|
-
// Parse await_durable
|
|
289
306
|
let write_opts = Self::write_options_from_kwargs(&kwargs)?;
|
|
307
|
+
let await_durable = Self::await_durable_from_kwargs(&kwargs)?;
|
|
290
308
|
|
|
291
309
|
block_on_result(async {
|
|
292
|
-
self
|
|
310
|
+
let handle = self
|
|
311
|
+
.inner
|
|
293
312
|
.put_with_options(key.as_bytes(), value.as_bytes(), &put_opts, &write_opts)
|
|
294
|
-
.await
|
|
313
|
+
.await?;
|
|
314
|
+
if await_durable {
|
|
315
|
+
handle.await_durable().await?;
|
|
316
|
+
}
|
|
317
|
+
Ok(())
|
|
295
318
|
})?;
|
|
296
319
|
self.increment_metric("db.put_with_options.count");
|
|
297
320
|
|
|
@@ -307,15 +330,14 @@ impl Database {
|
|
|
307
330
|
return Err(invalid_argument_error("key cannot be empty"));
|
|
308
331
|
}
|
|
309
332
|
|
|
310
|
-
let write_opts = WriteOptions
|
|
311
|
-
await_durable: true,
|
|
312
|
-
seqnum: 0,
|
|
313
|
-
};
|
|
333
|
+
let write_opts = WriteOptions::default();
|
|
314
334
|
|
|
315
335
|
block_on_result(async {
|
|
316
|
-
self
|
|
336
|
+
let handle = self
|
|
337
|
+
.inner
|
|
317
338
|
.delete_with_options(key.as_bytes(), &write_opts)
|
|
318
|
-
.await
|
|
339
|
+
.await?;
|
|
340
|
+
handle.await_durable().await
|
|
319
341
|
})?;
|
|
320
342
|
self.increment_metric("db.delete.count");
|
|
321
343
|
|
|
@@ -333,11 +355,17 @@ impl Database {
|
|
|
333
355
|
}
|
|
334
356
|
|
|
335
357
|
let write_opts = Self::write_options_from_kwargs(&kwargs)?;
|
|
358
|
+
let await_durable = Self::await_durable_from_kwargs(&kwargs)?;
|
|
336
359
|
|
|
337
360
|
block_on_result(async {
|
|
338
|
-
self
|
|
361
|
+
let handle = self
|
|
362
|
+
.inner
|
|
339
363
|
.delete_with_options(key.as_bytes(), &write_opts)
|
|
340
|
-
.await
|
|
364
|
+
.await?;
|
|
365
|
+
if await_durable {
|
|
366
|
+
handle.await_durable().await?;
|
|
367
|
+
}
|
|
368
|
+
Ok(())
|
|
341
369
|
})?;
|
|
342
370
|
self.increment_metric("db.delete_with_options.count");
|
|
343
371
|
|
|
@@ -467,7 +495,7 @@ impl Database {
|
|
|
467
495
|
let opts = ScanOptions::default();
|
|
468
496
|
let iter = block_on_result(async {
|
|
469
497
|
self.inner
|
|
470
|
-
.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
498
|
+
.scan_prefix_with_options(prefix.as_bytes(), .., &opts)
|
|
471
499
|
.await
|
|
472
500
|
})?;
|
|
473
501
|
|
|
@@ -534,9 +562,10 @@ impl Database {
|
|
|
534
562
|
};
|
|
535
563
|
}
|
|
536
564
|
|
|
565
|
+
let subrange = prefix_subrange_from_kwargs(&kwargs)?;
|
|
537
566
|
let iter = block_on_result(async {
|
|
538
567
|
self.inner
|
|
539
|
-
.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
568
|
+
.scan_prefix_with_options(prefix.as_bytes(), subrange, &opts)
|
|
540
569
|
.await
|
|
541
570
|
})?;
|
|
542
571
|
|
|
@@ -549,7 +578,10 @@ impl Database {
|
|
|
549
578
|
/// * `batch` - The WriteBatch to write
|
|
550
579
|
pub fn write(&self, batch: &WriteBatch) -> Result<(), Error> {
|
|
551
580
|
let batch_inner = batch.take()?;
|
|
552
|
-
block_on_result(async {
|
|
581
|
+
block_on_result(async {
|
|
582
|
+
let handle = self.inner.write(batch_inner).await?;
|
|
583
|
+
handle.await_durable().await
|
|
584
|
+
})?;
|
|
553
585
|
Ok(())
|
|
554
586
|
}
|
|
555
587
|
|
|
@@ -560,13 +592,19 @@ impl Database {
|
|
|
560
592
|
/// * `kwargs` - Keyword arguments (await_durable, seqnum)
|
|
561
593
|
pub fn write_with_options(&self, batch: &WriteBatch, kwargs: RHash) -> Result<(), Error> {
|
|
562
594
|
let write_opts = Self::write_options_from_kwargs(&kwargs)?;
|
|
595
|
+
let await_durable = Self::await_durable_from_kwargs(&kwargs)?;
|
|
563
596
|
|
|
564
597
|
let batch_inner = batch.take()?;
|
|
565
598
|
|
|
566
599
|
block_on_result(async {
|
|
567
|
-
self
|
|
600
|
+
let handle = self
|
|
601
|
+
.inner
|
|
568
602
|
.write_with_options(batch_inner, &write_opts)
|
|
569
|
-
.await
|
|
603
|
+
.await?;
|
|
604
|
+
if await_durable {
|
|
605
|
+
handle.await_durable().await?;
|
|
606
|
+
}
|
|
607
|
+
Ok(())
|
|
570
608
|
})?;
|
|
571
609
|
|
|
572
610
|
Ok(())
|
|
@@ -583,16 +621,14 @@ impl Database {
|
|
|
583
621
|
}
|
|
584
622
|
|
|
585
623
|
let merge_opts = MergeOptions { ttl: Ttl::Default };
|
|
586
|
-
|
|
587
|
-
let write_opts = WriteOptions {
|
|
588
|
-
await_durable: true,
|
|
589
|
-
seqnum: 0,
|
|
590
|
-
};
|
|
624
|
+
let write_opts = WriteOptions::default();
|
|
591
625
|
|
|
592
626
|
block_on_result(async {
|
|
593
|
-
self
|
|
627
|
+
let handle = self
|
|
628
|
+
.inner
|
|
594
629
|
.merge_with_options(key.as_bytes(), value.as_bytes(), &merge_opts, &write_opts)
|
|
595
|
-
.await
|
|
630
|
+
.await?;
|
|
631
|
+
handle.await_durable().await
|
|
596
632
|
})?;
|
|
597
633
|
|
|
598
634
|
Ok(())
|
|
@@ -617,17 +653,23 @@ impl Database {
|
|
|
617
653
|
let ttl = get_optional::<u64>(&kwargs, "ttl")?;
|
|
618
654
|
let merge_opts = MergeOptions {
|
|
619
655
|
ttl: match ttl {
|
|
620
|
-
Some(ms) => Ttl::
|
|
656
|
+
Some(ms) => Ttl::ExpireAfterMillis(ms),
|
|
621
657
|
None => Ttl::Default,
|
|
622
658
|
},
|
|
623
659
|
};
|
|
624
660
|
|
|
625
661
|
let write_opts = Self::write_options_from_kwargs(&kwargs)?;
|
|
662
|
+
let await_durable = Self::await_durable_from_kwargs(&kwargs)?;
|
|
626
663
|
|
|
627
664
|
block_on_result(async {
|
|
628
|
-
self
|
|
665
|
+
let handle = self
|
|
666
|
+
.inner
|
|
629
667
|
.merge_with_options(key.as_bytes(), value.as_bytes(), &merge_opts, &write_opts)
|
|
630
|
-
.await
|
|
668
|
+
.await?;
|
|
669
|
+
if await_durable {
|
|
670
|
+
handle.await_durable().await?;
|
|
671
|
+
}
|
|
672
|
+
Ok(())
|
|
631
673
|
})?;
|
|
632
674
|
|
|
633
675
|
Ok(())
|
|
@@ -707,6 +749,19 @@ impl Database {
|
|
|
707
749
|
Ok(())
|
|
708
750
|
}
|
|
709
751
|
|
|
752
|
+
/// Flush the database, choosing the flush type (SlateDB >= 0.16.0).
|
|
753
|
+
///
|
|
754
|
+
/// # Arguments
|
|
755
|
+
/// * `flush_type` - "wal" to flush the WAL, or "memtable" to freeze and
|
|
756
|
+
/// flush memtable entries to object storage.
|
|
757
|
+
pub fn flush_with_options(&self, flush_type: String) -> Result<(), Error> {
|
|
758
|
+
let options = FlushOptions {
|
|
759
|
+
flush_type: Self::parse_flush_type(&flush_type)?,
|
|
760
|
+
};
|
|
761
|
+
block_on_result(async { self.inner.flush_with_options(options).await })?;
|
|
762
|
+
Ok(())
|
|
763
|
+
}
|
|
764
|
+
|
|
710
765
|
/// Return the database metrics registry.
|
|
711
766
|
pub fn metrics(&self) -> Result<Metrics, Error> {
|
|
712
767
|
Ok(Metrics::new(self.metrics.clone()))
|
|
@@ -717,6 +772,22 @@ impl Database {
|
|
|
717
772
|
block_on_result(async { self.inner.close().await })?;
|
|
718
773
|
Ok(())
|
|
719
774
|
}
|
|
775
|
+
|
|
776
|
+
/// Close the database, controlling the final flush (SlateDB >= 0.16.0).
|
|
777
|
+
///
|
|
778
|
+
/// # Arguments
|
|
779
|
+
/// * `flush_type` - `Some("wal"|"memtable")` to flush before closing, or
|
|
780
|
+
/// `None` to close without a final flush (writes that are not yet durable
|
|
781
|
+
/// may be lost).
|
|
782
|
+
pub fn close_with_options(&self, flush_type: Option<String>) -> Result<(), Error> {
|
|
783
|
+
let flush_type = match flush_type {
|
|
784
|
+
Some(name) => Some(Self::parse_flush_type(&name)?),
|
|
785
|
+
None => None,
|
|
786
|
+
};
|
|
787
|
+
let options = CloseOptions { flush_type };
|
|
788
|
+
block_on_result(async { self.inner.close_with_options(options).await })?;
|
|
789
|
+
Ok(())
|
|
790
|
+
}
|
|
720
791
|
}
|
|
721
792
|
|
|
722
793
|
/// Define the Database class on the SlateDb module.
|
|
@@ -771,9 +842,17 @@ pub fn define_database_class(ruby: &Ruby, module: &magnus::RModule) -> Result<()
|
|
|
771
842
|
"_create_checkpoint",
|
|
772
843
|
method!(Database::create_checkpoint, 1),
|
|
773
844
|
)?;
|
|
774
|
-
class.define_method("
|
|
845
|
+
class.define_method("_flush", method!(Database::flush, 0))?;
|
|
846
|
+
class.define_method(
|
|
847
|
+
"_flush_with_options",
|
|
848
|
+
method!(Database::flush_with_options, 1),
|
|
849
|
+
)?;
|
|
775
850
|
class.define_method("_metrics", method!(Database::metrics, 0))?;
|
|
776
|
-
class.define_method("
|
|
851
|
+
class.define_method("_close", method!(Database::close, 0))?;
|
|
852
|
+
class.define_method(
|
|
853
|
+
"_close_with_options",
|
|
854
|
+
method!(Database::close_with_options, 1),
|
|
855
|
+
)?;
|
|
777
856
|
|
|
778
857
|
Ok(())
|
|
779
858
|
}
|
data/ext/slatedb/src/reader.rs
CHANGED
|
@@ -4,12 +4,13 @@ use magnus::prelude::*;
|
|
|
4
4
|
use magnus::{function, method, Error, RHash, Ruby};
|
|
5
5
|
use slatedb::config::{DbReaderOptions, DurabilityLevel, ReadOptions, ScanOptions};
|
|
6
6
|
use slatedb::DbReader;
|
|
7
|
+
use slatedb::DbReaderMode;
|
|
7
8
|
use slatedb::IterationOrder;
|
|
8
9
|
|
|
9
10
|
use crate::errors::invalid_argument_error;
|
|
10
11
|
use crate::iterator::Iterator;
|
|
11
12
|
use crate::runtime::block_on_result;
|
|
12
|
-
use crate::utils::{get_optional, resolve_object_store};
|
|
13
|
+
use crate::utils::{get_optional, prefix_subrange_from_kwargs, resolve_object_store};
|
|
13
14
|
|
|
14
15
|
/// Ruby wrapper for SlateDB Reader.
|
|
15
16
|
///
|
|
@@ -26,11 +27,20 @@ impl Reader {
|
|
|
26
27
|
/// # Arguments
|
|
27
28
|
/// * `path` - The path identifier for the database
|
|
28
29
|
/// * `url` - Optional object store URL
|
|
29
|
-
/// * `checkpoint_id` - Optional checkpoint UUID to read at
|
|
30
|
+
/// * `checkpoint_id` - Optional checkpoint UUID to read at. When set, the reader
|
|
31
|
+
/// is pinned to that checkpoint ([`DbReaderMode::Checkpoint`]).
|
|
30
32
|
/// * `kwargs` - Additional options (manifest_poll_interval, checkpoint_lifetime,
|
|
31
|
-
/// max_memtable_bytes, skip_wal_replay, cache_root, max_open_file_handles
|
|
33
|
+
/// max_memtable_bytes, skip_wal_replay, cache_root, max_open_file_handles,
|
|
34
|
+
/// follow_latest).
|
|
32
35
|
/// The local disk cache (and therefore `max_open_file_handles`) is only active
|
|
33
36
|
/// when `cache_root` is set.
|
|
37
|
+
///
|
|
38
|
+
/// When neither `checkpoint_id` nor `follow_latest` is given the reader defaults
|
|
39
|
+
/// to [`DbReaderMode::ManagedCheckpoint`], creating and periodically refreshing
|
|
40
|
+
/// its own checkpoint so garbage collection cannot delete objects out from under
|
|
41
|
+
/// it. Setting `follow_latest: true` selects [`DbReaderMode::FollowLatest`], which
|
|
42
|
+
/// tails the latest manifest without writing any checkpoint — useful for read-only
|
|
43
|
+
/// or mirrored databases, at the cost of no protection from garbage collection.
|
|
34
44
|
pub fn open(
|
|
35
45
|
path: String,
|
|
36
46
|
url: Option<String>,
|
|
@@ -46,16 +56,24 @@ impl Reader {
|
|
|
46
56
|
let skip_wal_replay = get_optional::<bool>(&kwargs, "skip_wal_replay")?;
|
|
47
57
|
let max_open_file_handles = get_optional::<usize>(&kwargs, "max_open_file_handles")?;
|
|
48
58
|
let cache_root = get_optional::<String>(&kwargs, "cache_root")?;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
59
|
+
let follow_latest = get_optional::<bool>(&kwargs, "follow_latest")?.unwrap_or(false);
|
|
60
|
+
|
|
61
|
+
// Resolve the reader mode from checkpoint_id / follow_latest.
|
|
62
|
+
let mode = match checkpoint_id {
|
|
63
|
+
Some(id_str) => {
|
|
64
|
+
if follow_latest {
|
|
65
|
+
return Err(invalid_argument_error(
|
|
66
|
+
"checkpoint_id and follow_latest are mutually exclusive",
|
|
67
|
+
));
|
|
68
|
+
}
|
|
69
|
+
let uuid = uuid::Uuid::parse_str(&id_str).map_err(|e| {
|
|
54
70
|
invalid_argument_error(&format!("invalid checkpoint_id: {}", e))
|
|
55
|
-
})
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
71
|
+
})?;
|
|
72
|
+
DbReaderMode::Checkpoint(uuid)
|
|
73
|
+
}
|
|
74
|
+
None if follow_latest => DbReaderMode::FollowLatest,
|
|
75
|
+
None => DbReaderMode::ManagedCheckpoint,
|
|
76
|
+
};
|
|
59
77
|
|
|
60
78
|
let reader = block_on_result(async {
|
|
61
79
|
let object_store: Arc<dyn slatedb::object_store::ObjectStore> =
|
|
@@ -85,7 +103,7 @@ impl Reader {
|
|
|
85
103
|
if let Some(max_handles) = max_open_file_handles {
|
|
86
104
|
options.object_store_cache_options.max_open_file_handles = max_handles;
|
|
87
105
|
}
|
|
88
|
-
DbReader::open(path, object_store,
|
|
106
|
+
DbReader::open(path, object_store, mode, options).await
|
|
89
107
|
})?;
|
|
90
108
|
|
|
91
109
|
Ok(Self {
|
|
@@ -239,7 +257,8 @@ impl Reader {
|
|
|
239
257
|
return Err(invalid_argument_error("prefix cannot be empty"));
|
|
240
258
|
}
|
|
241
259
|
|
|
242
|
-
let iter =
|
|
260
|
+
let iter =
|
|
261
|
+
block_on_result(async { self.inner.scan_prefix(prefix.as_bytes(), ..).await })?;
|
|
243
262
|
|
|
244
263
|
Ok(Iterator::new(iter))
|
|
245
264
|
}
|
|
@@ -297,9 +316,10 @@ impl Reader {
|
|
|
297
316
|
};
|
|
298
317
|
}
|
|
299
318
|
|
|
319
|
+
let subrange = prefix_subrange_from_kwargs(&kwargs)?;
|
|
300
320
|
let iter = block_on_result(async {
|
|
301
321
|
self.inner
|
|
302
|
-
.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
322
|
+
.scan_prefix_with_options(prefix.as_bytes(), subrange, &opts)
|
|
303
323
|
.await
|
|
304
324
|
})?;
|
|
305
325
|
|
data/ext/slatedb/src/snapshot.rs
CHANGED
|
@@ -10,7 +10,7 @@ use slatedb::IterationOrder;
|
|
|
10
10
|
use crate::errors::{closed_error, invalid_argument_error};
|
|
11
11
|
use crate::iterator::Iterator;
|
|
12
12
|
use crate::runtime::block_on_result;
|
|
13
|
-
use crate::utils::get_optional;
|
|
13
|
+
use crate::utils::{get_optional, prefix_subrange_from_kwargs};
|
|
14
14
|
|
|
15
15
|
/// Ruby wrapper for SlateDB Snapshot.
|
|
16
16
|
///
|
|
@@ -190,7 +190,7 @@ impl Snapshot {
|
|
|
190
190
|
.as_ref()
|
|
191
191
|
.ok_or_else(|| closed_error("snapshot is closed"))?;
|
|
192
192
|
|
|
193
|
-
let iter = block_on_result(async { snapshot.scan_prefix(prefix.as_bytes()).await })?;
|
|
193
|
+
let iter = block_on_result(async { snapshot.scan_prefix(prefix.as_bytes(), ..).await })?;
|
|
194
194
|
|
|
195
195
|
Ok(Iterator::new(iter))
|
|
196
196
|
}
|
|
@@ -248,6 +248,8 @@ impl Snapshot {
|
|
|
248
248
|
};
|
|
249
249
|
}
|
|
250
250
|
|
|
251
|
+
let subrange = prefix_subrange_from_kwargs(&kwargs)?;
|
|
252
|
+
|
|
251
253
|
let guard = self.inner.borrow();
|
|
252
254
|
let snapshot = guard
|
|
253
255
|
.as_ref()
|
|
@@ -255,7 +257,7 @@ impl Snapshot {
|
|
|
255
257
|
|
|
256
258
|
let iter = block_on_result(async {
|
|
257
259
|
snapshot
|
|
258
|
-
.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
260
|
+
.scan_prefix_with_options(prefix.as_bytes(), subrange, &opts)
|
|
259
261
|
.await
|
|
260
262
|
})?;
|
|
261
263
|
|
|
@@ -11,7 +11,7 @@ use slatedb::IterationOrder;
|
|
|
11
11
|
use crate::errors::{closed_error, invalid_argument_error, map_error};
|
|
12
12
|
use crate::iterator::Iterator;
|
|
13
13
|
use crate::runtime::block_on_result;
|
|
14
|
-
use crate::utils::get_optional;
|
|
14
|
+
use crate::utils::{get_optional, prefix_subrange_from_kwargs};
|
|
15
15
|
|
|
16
16
|
/// Ruby wrapper for SlateDB Transaction.
|
|
17
17
|
///
|
|
@@ -109,7 +109,7 @@ impl Transaction {
|
|
|
109
109
|
let ttl = get_optional::<u64>(&kwargs, "ttl")?;
|
|
110
110
|
let put_opts = PutOptions {
|
|
111
111
|
ttl: match ttl {
|
|
112
|
-
Some(ms) => Ttl::
|
|
112
|
+
Some(ms) => Ttl::ExpireAfterMillis(ms),
|
|
113
113
|
None => Ttl::Default,
|
|
114
114
|
},
|
|
115
115
|
};
|
|
@@ -172,7 +172,7 @@ impl Transaction {
|
|
|
172
172
|
let ttl = get_optional::<u64>(&kwargs, "ttl")?;
|
|
173
173
|
let merge_opts = MergeOptions {
|
|
174
174
|
ttl: match ttl {
|
|
175
|
-
Some(ms) => Ttl::
|
|
175
|
+
Some(ms) => Ttl::ExpireAfterMillis(ms),
|
|
176
176
|
None => Ttl::Default,
|
|
177
177
|
},
|
|
178
178
|
};
|
|
@@ -295,7 +295,7 @@ impl Transaction {
|
|
|
295
295
|
.as_ref()
|
|
296
296
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
297
297
|
|
|
298
|
-
let iter = block_on_result(async { txn.scan_prefix(prefix.as_bytes()).await })?;
|
|
298
|
+
let iter = block_on_result(async { txn.scan_prefix(prefix.as_bytes(), ..).await })?;
|
|
299
299
|
|
|
300
300
|
Ok(Iterator::new(iter))
|
|
301
301
|
}
|
|
@@ -353,13 +353,16 @@ impl Transaction {
|
|
|
353
353
|
};
|
|
354
354
|
}
|
|
355
355
|
|
|
356
|
+
let subrange = prefix_subrange_from_kwargs(&kwargs)?;
|
|
357
|
+
|
|
356
358
|
let guard = self.inner.borrow();
|
|
357
359
|
let txn = guard
|
|
358
360
|
.as_ref()
|
|
359
361
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
360
362
|
|
|
361
363
|
let iter = block_on_result(async {
|
|
362
|
-
txn.scan_prefix_with_options(prefix.as_bytes(), &opts)
|
|
364
|
+
txn.scan_prefix_with_options(prefix.as_bytes(), subrange, &opts)
|
|
365
|
+
.await
|
|
363
366
|
})?;
|
|
364
367
|
|
|
365
368
|
Ok(Iterator::new(iter))
|
|
@@ -392,7 +395,16 @@ impl Transaction {
|
|
|
392
395
|
.take()
|
|
393
396
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
394
397
|
|
|
395
|
-
block_on_result(async {
|
|
398
|
+
block_on_result(async {
|
|
399
|
+
// As of SlateDB 0.16.0, commit returns as soon as the write is in
|
|
400
|
+
// memory and hands back an optional WriteHandle (None for an empty
|
|
401
|
+
// or read-only transaction). Await durability to preserve the
|
|
402
|
+
// previous blocking-commit behavior.
|
|
403
|
+
if let Some(handle) = txn.commit().await? {
|
|
404
|
+
handle.await_durable().await?;
|
|
405
|
+
}
|
|
406
|
+
Ok(())
|
|
407
|
+
})?;
|
|
396
408
|
Ok(())
|
|
397
409
|
}
|
|
398
410
|
|
|
@@ -400,10 +412,7 @@ impl Transaction {
|
|
|
400
412
|
pub fn commit_with_options(&self, kwargs: RHash) -> Result<(), Error> {
|
|
401
413
|
let await_durable = get_optional::<bool>(&kwargs, "await_durable")?.unwrap_or(true);
|
|
402
414
|
let seqnum = get_optional::<u64>(&kwargs, "seqnum")?.unwrap_or(0);
|
|
403
|
-
let write_opts = WriteOptions {
|
|
404
|
-
await_durable,
|
|
405
|
-
seqnum,
|
|
406
|
-
};
|
|
415
|
+
let write_opts = WriteOptions { seqnum };
|
|
407
416
|
|
|
408
417
|
let txn = self
|
|
409
418
|
.inner
|
|
@@ -411,7 +420,14 @@ impl Transaction {
|
|
|
411
420
|
.take()
|
|
412
421
|
.ok_or_else(|| closed_error("transaction is closed"))?;
|
|
413
422
|
|
|
414
|
-
block_on_result(async {
|
|
423
|
+
block_on_result(async {
|
|
424
|
+
if let Some(handle) = txn.commit_with_options(&write_opts).await? {
|
|
425
|
+
if await_durable {
|
|
426
|
+
handle.await_durable().await?;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
Ok(())
|
|
430
|
+
})?;
|
|
415
431
|
Ok(())
|
|
416
432
|
}
|
|
417
433
|
|
data/ext/slatedb/src/utils.rs
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
|
+
use std::ops::Bound;
|
|
1
2
|
use std::sync::Arc;
|
|
2
3
|
|
|
3
4
|
use magnus::value::ReprValue;
|
|
4
5
|
use magnus::{Error, RHash, Ruby, TryConvert};
|
|
5
6
|
use slatedb::object_store::aws::AmazonS3Builder;
|
|
6
|
-
use slatedb::object_store::
|
|
7
|
-
use slatedb::{
|
|
7
|
+
use slatedb::object_store::prefix::PrefixStore;
|
|
8
|
+
use slatedb::object_store::{
|
|
9
|
+
parse_url_opts, Error as ObjectStoreError, ObjectStore, ObjectStoreScheme,
|
|
10
|
+
};
|
|
11
|
+
use slatedb::Error as SlateError;
|
|
8
12
|
use url::Url;
|
|
9
13
|
|
|
10
14
|
/// Helper to extract an optional value from an RHash
|
|
@@ -23,6 +27,30 @@ pub fn get_optional<T: TryConvert>(hash: &RHash, key: &str) -> Result<Option<T>,
|
|
|
23
27
|
}
|
|
24
28
|
}
|
|
25
29
|
|
|
30
|
+
/// A key sub-range for a prefix scan, expressed as owned byte bounds. This
|
|
31
|
+
/// implements `slatedb::bytes_range::ByteRangeBounds`, so it can be passed
|
|
32
|
+
/// straight to `scan_prefix`/`scan_prefix_with_options`.
|
|
33
|
+
pub type PrefixSubrange = (Bound<Vec<u8>>, Bound<Vec<u8>>);
|
|
34
|
+
|
|
35
|
+
/// Build a suffix sub-range (relative to a scan prefix) from optional
|
|
36
|
+
/// `subrange_from` / `subrange_to` keyword arguments.
|
|
37
|
+
///
|
|
38
|
+
/// The bounds are key *suffixes* appended to the prefix: `subrange_from` is an
|
|
39
|
+
/// inclusive lower bound and `subrange_to` is an exclusive upper bound. A
|
|
40
|
+
/// missing bound is unbounded, so an empty hash yields a full-prefix scan
|
|
41
|
+
/// (equivalent to `..`).
|
|
42
|
+
pub fn prefix_subrange_from_kwargs(hash: &RHash) -> Result<PrefixSubrange, Error> {
|
|
43
|
+
let start = match get_optional::<String>(hash, "subrange_from")? {
|
|
44
|
+
Some(s) => Bound::Included(s.into_bytes()),
|
|
45
|
+
None => Bound::Unbounded,
|
|
46
|
+
};
|
|
47
|
+
let end = match get_optional::<String>(hash, "subrange_to")? {
|
|
48
|
+
Some(s) => Bound::Excluded(s.into_bytes()),
|
|
49
|
+
None => Bound::Unbounded,
|
|
50
|
+
};
|
|
51
|
+
Ok((start, end))
|
|
52
|
+
}
|
|
53
|
+
|
|
26
54
|
/// Convert an object_store error to a SlateDB error
|
|
27
55
|
fn to_slate_error(e: ObjectStoreError) -> SlateError {
|
|
28
56
|
SlateError::unavailable(e.to_string())
|
|
@@ -52,8 +80,22 @@ pub fn resolve_object_store(url: &str) -> Result<Arc<dyn ObjectStore>, SlateErro
|
|
|
52
80
|
Ok(Arc::new(store))
|
|
53
81
|
}
|
|
54
82
|
_ => {
|
|
55
|
-
//
|
|
56
|
-
|
|
83
|
+
// SlateDB 0.14 changed `Db::resolve_object_store` to reject any URL
|
|
84
|
+
// that carries a path component (returning `InvalidObjectStorePath`),
|
|
85
|
+
// whereas 0.13 transparently wrapped it in a `PrefixStore`. Preserve
|
|
86
|
+
// the old, more forgiving behavior here so callers can keep passing a
|
|
87
|
+
// full location such as "file:///data/mydb" or "gs://bucket/prefix".
|
|
88
|
+
//
|
|
89
|
+
// Env keys are lowercased because `parse_url_opts` only recognizes
|
|
90
|
+
// lower-case option keys.
|
|
91
|
+
let env_vars = std::env::vars().map(|(k, v)| (k.to_ascii_lowercase(), v));
|
|
92
|
+
let (store, path) = parse_url_opts(&parsed_url, env_vars).map_err(to_slate_error)?;
|
|
93
|
+
let store: Arc<dyn ObjectStore> = Arc::from(store);
|
|
94
|
+
if path.as_ref().is_empty() {
|
|
95
|
+
Ok(store)
|
|
96
|
+
} else {
|
|
97
|
+
Ok(Arc::new(PrefixStore::new(store, path)))
|
|
98
|
+
}
|
|
57
99
|
}
|
|
58
100
|
}
|
|
59
101
|
}
|
|
@@ -49,7 +49,7 @@ impl WriteBatch {
|
|
|
49
49
|
let ttl = get_optional::<u64>(&kwargs, "ttl")?;
|
|
50
50
|
let put_opts = PutOptions {
|
|
51
51
|
ttl: match ttl {
|
|
52
|
-
Some(ms) => Ttl::
|
|
52
|
+
Some(ms) => Ttl::ExpireAfterMillis(ms),
|
|
53
53
|
None => Ttl::Default,
|
|
54
54
|
},
|
|
55
55
|
};
|
|
@@ -102,7 +102,7 @@ impl WriteBatch {
|
|
|
102
102
|
let ttl = get_optional::<u64>(&kwargs, "ttl")?;
|
|
103
103
|
let merge_opts = MergeOptions {
|
|
104
104
|
ttl: match ttl {
|
|
105
|
-
Some(ms) => Ttl::
|
|
105
|
+
Some(ms) => Ttl::ExpireAfterMillis(ms),
|
|
106
106
|
None => Ttl::Default,
|
|
107
107
|
},
|
|
108
108
|
};
|
data/lib/slatedb/admin.rb
CHANGED
|
@@ -118,5 +118,33 @@ module SlateDb
|
|
|
118
118
|
opts[:min_age] = min_age if min_age
|
|
119
119
|
_run_gc(opts)
|
|
120
120
|
end
|
|
121
|
+
|
|
122
|
+
# Delete a database, removing its objects from the object store
|
|
123
|
+
# (SlateDB >= 0.16.0).
|
|
124
|
+
#
|
|
125
|
+
# For a cloned database, any checkpoints it pinned in parent databases are
|
|
126
|
+
# stripped before its own objects are removed.
|
|
127
|
+
#
|
|
128
|
+
# By default this performs a dry run: it returns the object paths that
|
|
129
|
+
# *would* be deleted without removing anything. Pass +confirm: true+ to
|
|
130
|
+
# actually delete. A confirmed delete of a directory that is not a SlateDB
|
|
131
|
+
# database (no manifest and no in-progress deletion marker) is refused, so a
|
|
132
|
+
# mistyped path cannot wipe an unrelated directory. The operation is
|
|
133
|
+
# idempotent.
|
|
134
|
+
#
|
|
135
|
+
# @param confirm [Boolean] When false (the default), performs a dry run and
|
|
136
|
+
# returns the paths that would be deleted. When true, deletes and returns
|
|
137
|
+
# the paths that were removed.
|
|
138
|
+
# @return [Array<String>] Object paths deleted (or that would be deleted)
|
|
139
|
+
#
|
|
140
|
+
# @example Preview what would be deleted
|
|
141
|
+
# admin.delete_db.each { |path| puts path }
|
|
142
|
+
#
|
|
143
|
+
# @example Actually delete the database
|
|
144
|
+
# deleted = admin.delete_db(confirm: true)
|
|
145
|
+
#
|
|
146
|
+
def delete_db(confirm: false)
|
|
147
|
+
_delete_db(confirm)
|
|
148
|
+
end
|
|
121
149
|
end
|
|
122
150
|
end
|
data/lib/slatedb/database.rb
CHANGED
|
@@ -246,6 +246,11 @@ module SlateDb
|
|
|
246
246
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
247
247
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
248
248
|
# @param order [Symbol, String, nil] Iteration order (:asc/:ascending or :desc/:descending)
|
|
249
|
+
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
250
|
+
# prefix, to start scanning from (e.g. prefix "user:" with from "100"
|
|
251
|
+
# starts at "user:100"). Defaults to the start of the prefix.
|
|
252
|
+
# @param to [String, nil] Exclusive upper bound suffix, appended to the
|
|
253
|
+
# prefix, to stop scanning at. Defaults to the end of the prefix.
|
|
249
254
|
# @return [Iterator] An iterator over key-value pairs
|
|
250
255
|
#
|
|
251
256
|
# @example Scan all user keys
|
|
@@ -253,8 +258,13 @@ module SlateDb
|
|
|
253
258
|
# puts "#{key}: #{value}"
|
|
254
259
|
# end
|
|
255
260
|
#
|
|
261
|
+
# @example Scan a sub-range within a prefix
|
|
262
|
+
# # keys "user:100" (inclusive) up to "user:200" (exclusive)
|
|
263
|
+
# db.scan_prefix("user:", from: "100", to: "200")
|
|
264
|
+
#
|
|
256
265
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
257
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, order: nil,
|
|
266
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, order: nil,
|
|
267
|
+
from: nil, to: nil, &)
|
|
258
268
|
opts = scan_options(
|
|
259
269
|
durability_filter: durability_filter,
|
|
260
270
|
dirty: dirty,
|
|
@@ -263,6 +273,8 @@ module SlateDb
|
|
|
263
273
|
max_fetch_tasks: max_fetch_tasks,
|
|
264
274
|
order: order
|
|
265
275
|
)
|
|
276
|
+
opts[:subrange_from] = from if from
|
|
277
|
+
opts[:subrange_to] = to if to
|
|
266
278
|
|
|
267
279
|
iter = if opts.empty?
|
|
268
280
|
_scan_prefix(prefix)
|
|
@@ -494,5 +506,63 @@ module SlateDb
|
|
|
494
506
|
def metrics
|
|
495
507
|
_metrics
|
|
496
508
|
end
|
|
509
|
+
|
|
510
|
+
# Flush in-memory writes to object storage.
|
|
511
|
+
#
|
|
512
|
+
# With no arguments this flushes pending writes durably, matching the
|
|
513
|
+
# previous behavior. Pass +flush_type+ to control what is flushed
|
|
514
|
+
# (SlateDB >= 0.16.0).
|
|
515
|
+
#
|
|
516
|
+
# @param flush_type [Symbol, String, nil] Which data to flush: :wal to flush
|
|
517
|
+
# the write-ahead log, or :memtable to freeze and flush memtable entries
|
|
518
|
+
# to object storage. When nil (the default), SlateDB flushes the WAL if
|
|
519
|
+
# enabled, otherwise the memtable.
|
|
520
|
+
# @return [void]
|
|
521
|
+
#
|
|
522
|
+
# @example Flush pending writes
|
|
523
|
+
# db.flush
|
|
524
|
+
#
|
|
525
|
+
# @example Explicitly flush the WAL
|
|
526
|
+
# db.flush(flush_type: :wal)
|
|
527
|
+
#
|
|
528
|
+
def flush(flush_type: nil)
|
|
529
|
+
if flush_type.nil?
|
|
530
|
+
_flush
|
|
531
|
+
else
|
|
532
|
+
_flush_with_options(flush_type.to_s)
|
|
533
|
+
end
|
|
534
|
+
end
|
|
535
|
+
|
|
536
|
+
# Close the database.
|
|
537
|
+
#
|
|
538
|
+
# By default the active memtable is flushed before closing (matching the
|
|
539
|
+
# previous behavior). Pass +flush: false+ to close without a final flush,
|
|
540
|
+
# in which case writes that are not yet durable may be lost
|
|
541
|
+
# (SlateDB >= 0.16.0).
|
|
542
|
+
#
|
|
543
|
+
# @param flush [Boolean] Whether to flush before closing (default: true)
|
|
544
|
+
# @param flush_type [Symbol, String, nil] When flushing, which data to
|
|
545
|
+
# flush: :wal or :memtable. Defaults to :memtable. Ignored when
|
|
546
|
+
# +flush: false+.
|
|
547
|
+
# @return [void]
|
|
548
|
+
#
|
|
549
|
+
# @example Close (flushing the memtable first)
|
|
550
|
+
# db.close
|
|
551
|
+
#
|
|
552
|
+
# @example Close without flushing
|
|
553
|
+
# db.close(flush: false)
|
|
554
|
+
#
|
|
555
|
+
# @example Flush the WAL before closing
|
|
556
|
+
# db.close(flush_type: :wal)
|
|
557
|
+
#
|
|
558
|
+
def close(flush: true, flush_type: nil)
|
|
559
|
+
if !flush
|
|
560
|
+
_close_with_options(nil)
|
|
561
|
+
elsif flush_type.nil?
|
|
562
|
+
_close
|
|
563
|
+
else
|
|
564
|
+
_close_with_options(flush_type.to_s)
|
|
565
|
+
end
|
|
566
|
+
end
|
|
497
567
|
end
|
|
498
568
|
end
|
data/lib/slatedb/reader.rb
CHANGED
|
@@ -7,7 +7,14 @@ module SlateDb
|
|
|
7
7
|
#
|
|
8
8
|
# @param path [String] The path identifier for the database
|
|
9
9
|
# @param url [String, nil] Optional object store URL
|
|
10
|
-
# @param checkpoint_id [String, nil] Optional checkpoint UUID to read at
|
|
10
|
+
# @param checkpoint_id [String, nil] Optional checkpoint UUID to read at. When
|
|
11
|
+
# given, the reader is pinned to that checkpoint and does not follow new writes.
|
|
12
|
+
# @param follow_latest [Boolean, nil] When true, the reader tails the latest
|
|
13
|
+
# manifest without creating or maintaining its own checkpoint. This performs no
|
|
14
|
+
# object-store writes but provides no protection from garbage collection, so it
|
|
15
|
+
# is best suited to read-only or mirrored databases. Mutually exclusive with
|
|
16
|
+
# +checkpoint_id+. When neither is set (the default), the reader manages its own
|
|
17
|
+
# checkpoint and refreshes it periodically. (Requires SlateDB >= 0.15.0)
|
|
11
18
|
# @param manifest_poll_interval [Integer, nil] Poll interval in milliseconds
|
|
12
19
|
# @param checkpoint_lifetime [Integer, nil] Checkpoint lifetime in milliseconds
|
|
13
20
|
# @param max_memtable_bytes [Integer, nil] Maximum memtable size in bytes
|
|
@@ -35,16 +42,20 @@ module SlateDb
|
|
|
35
42
|
# @example Open at a specific checkpoint
|
|
36
43
|
# reader = SlateDb::Reader.open("/tmp/mydb", checkpoint_id: "uuid-here")
|
|
37
44
|
#
|
|
45
|
+
# @example Follow the latest state without managing a checkpoint
|
|
46
|
+
# reader = SlateDb::Reader.open("/tmp/mydb", follow_latest: true)
|
|
47
|
+
#
|
|
38
48
|
# @example Enable the on-disk cache and cap its open file handles
|
|
39
49
|
# reader = SlateDb::Reader.open("/tmp/mydb",
|
|
40
50
|
# cache_root: "/var/cache/slatedb",
|
|
41
51
|
# max_open_file_handles: 256)
|
|
42
52
|
#
|
|
43
|
-
def open(path, url: nil, checkpoint_id: nil,
|
|
53
|
+
def open(path, url: nil, checkpoint_id: nil, follow_latest: nil,
|
|
44
54
|
manifest_poll_interval: nil, checkpoint_lifetime: nil,
|
|
45
55
|
max_memtable_bytes: nil, cache_root: nil, max_open_file_handles: nil,
|
|
46
56
|
merge_operator: nil)
|
|
47
57
|
opts = {}
|
|
58
|
+
opts[:follow_latest] = follow_latest unless follow_latest.nil?
|
|
48
59
|
opts[:manifest_poll_interval] = manifest_poll_interval if manifest_poll_interval
|
|
49
60
|
opts[:checkpoint_lifetime] = checkpoint_lifetime if checkpoint_lifetime
|
|
50
61
|
opts[:max_memtable_bytes] = max_memtable_bytes if max_memtable_bytes
|
|
@@ -127,16 +138,25 @@ module SlateDb
|
|
|
127
138
|
# @param read_ahead_bytes [Integer, nil] Number of bytes to read ahead
|
|
128
139
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
129
140
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
141
|
+
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
142
|
+
# prefix, to start scanning from (e.g. prefix "user:" with from "100"
|
|
143
|
+
# starts at "user:100"). Defaults to the start of the prefix.
|
|
144
|
+
# @param to [String, nil] Exclusive upper bound suffix, appended to the
|
|
145
|
+
# prefix, to stop scanning at. Defaults to the end of the prefix.
|
|
130
146
|
# @return [Iterator] An iterator over key-value pairs
|
|
131
147
|
#
|
|
132
148
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
133
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
134
|
-
|
|
135
|
-
opts
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
149
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
150
|
+
from: nil, to: nil, &)
|
|
151
|
+
opts = {
|
|
152
|
+
durability_filter: durability_filter&.to_s,
|
|
153
|
+
dirty: dirty,
|
|
154
|
+
read_ahead_bytes: read_ahead_bytes,
|
|
155
|
+
cache_blocks: cache_blocks,
|
|
156
|
+
max_fetch_tasks: max_fetch_tasks,
|
|
157
|
+
subrange_from: from,
|
|
158
|
+
subrange_to: to
|
|
159
|
+
}.compact
|
|
140
160
|
|
|
141
161
|
iter = if opts.empty?
|
|
142
162
|
_scan_prefix(prefix)
|
data/lib/slatedb/snapshot.rb
CHANGED
|
@@ -59,16 +59,24 @@ module SlateDb
|
|
|
59
59
|
# @param read_ahead_bytes [Integer, nil] Number of bytes to read ahead
|
|
60
60
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
61
61
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
62
|
+
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
63
|
+
# prefix, to start scanning from. Defaults to the start of the prefix.
|
|
64
|
+
# @param to [String, nil] Exclusive upper bound suffix, appended to the
|
|
65
|
+
# prefix, to stop scanning at. Defaults to the end of the prefix.
|
|
62
66
|
# @return [Iterator] An iterator over key-value pairs
|
|
63
67
|
#
|
|
64
68
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
65
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
66
|
-
|
|
67
|
-
opts
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
70
|
+
from: nil, to: nil, &)
|
|
71
|
+
opts = {
|
|
72
|
+
durability_filter: durability_filter&.to_s,
|
|
73
|
+
dirty: dirty,
|
|
74
|
+
read_ahead_bytes: read_ahead_bytes,
|
|
75
|
+
cache_blocks: cache_blocks,
|
|
76
|
+
max_fetch_tasks: max_fetch_tasks,
|
|
77
|
+
subrange_from: from,
|
|
78
|
+
subrange_to: to
|
|
79
|
+
}.compact
|
|
72
80
|
|
|
73
81
|
iter = if opts.empty?
|
|
74
82
|
_scan_prefix(prefix)
|
data/lib/slatedb/transaction.rb
CHANGED
|
@@ -98,16 +98,24 @@ module SlateDb
|
|
|
98
98
|
# @param read_ahead_bytes [Integer, nil] Number of bytes to read ahead
|
|
99
99
|
# @param cache_blocks [Boolean, nil] Whether to cache blocks
|
|
100
100
|
# @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
|
|
101
|
+
# @param from [String, nil] Inclusive lower bound suffix, appended to the
|
|
102
|
+
# prefix, to start scanning from. Defaults to the start of the prefix.
|
|
103
|
+
# @param to [String, nil] Exclusive upper bound suffix, appended to the
|
|
104
|
+
# prefix, to stop scanning at. Defaults to the end of the prefix.
|
|
101
105
|
# @return [Iterator] An iterator over key-value pairs
|
|
102
106
|
#
|
|
103
107
|
def scan_prefix(prefix, durability_filter: nil, dirty: nil,
|
|
104
|
-
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
105
|
-
|
|
106
|
-
opts
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
108
|
+
read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
|
|
109
|
+
from: nil, to: nil, &)
|
|
110
|
+
opts = {
|
|
111
|
+
durability_filter: durability_filter&.to_s,
|
|
112
|
+
dirty: dirty,
|
|
113
|
+
read_ahead_bytes: read_ahead_bytes,
|
|
114
|
+
cache_blocks: cache_blocks,
|
|
115
|
+
max_fetch_tasks: max_fetch_tasks,
|
|
116
|
+
subrange_from: from,
|
|
117
|
+
subrange_to: to
|
|
118
|
+
}.compact
|
|
111
119
|
|
|
112
120
|
iter = if opts.empty?
|
|
113
121
|
_scan_prefix(prefix)
|
data/lib/slatedb/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: slatedb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- SlateDB Contributors
|
|
@@ -137,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
137
137
|
- !ruby/object:Gem::Version
|
|
138
138
|
version: '0'
|
|
139
139
|
requirements: []
|
|
140
|
-
rubygems_version: 4.0.
|
|
140
|
+
rubygems_version: 4.0.20
|
|
141
141
|
specification_version: 4
|
|
142
142
|
summary: Ruby bindings for SlateDB
|
|
143
143
|
test_files: []
|