slatedb 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +150 -44
- data/ext/slatedb/Cargo.toml +7 -8
- data/ext/slatedb/src/checkpoint.rs +47 -0
- data/ext/slatedb/src/database.rs +61 -19
- data/ext/slatedb/src/lib.rs +2 -0
- data/ext/slatedb/src/merge_ops.rs +2 -185
- data/ext/slatedb/src/reader.rs +21 -6
- data/ext/slatedb/src/runtime.rs +1 -52
- data/ext/slatedb/src/snapshot.rs +4 -0
- data/ext/slatedb/src/transaction.rs +4 -0
- data/lib/slatedb/database.rb +51 -27
- data/lib/slatedb/reader.rb +38 -27
- data/lib/slatedb/snapshot.rb +17 -9
- data/lib/slatedb/transaction.rb +17 -9
- data/lib/slatedb/version.rb +1 -1
- metadata +6 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d78fa73dbdaa8c8bd6bdacebc65b3f69b3df642eab3f7e8d7bdd08bde68b0ba0
|
|
4
|
+
data.tar.gz: 9dd7197dea492b9a146053f1aadad0bc233f258d5cceea2aadda64ad8c69a5c6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 783e32df3d81361f30ab91003b9fa10848e342b548219eb4b622be0d760c2ba3d6a5f1211b455ea7ccd7d05e6f13dc793700878c6ac370cd4204a92de4b8cc8d
|
|
7
|
+
data.tar.gz: 5239793e435456d610b5a2eea877e58873f4d48a7e80f47dcc9b191cc82857d0f5669dd6780e0d26997b616e4bc4e501ac917b7f4860dce9d832269969689177
|
data/README.md
CHANGED
|
@@ -30,6 +30,38 @@ gem install slatedb
|
|
|
30
30
|
|
|
31
31
|
Precompiled native gems are published for common Linux and Darwin platforms. Building from source still requires a Rust toolchain.
|
|
32
32
|
|
|
33
|
+
## Upgrading to SlateDB 0.18.1
|
|
34
|
+
|
|
35
|
+
This update moves the Rust engine from 0.17.0 to 0.18.1. Existing database files
|
|
36
|
+
remain readable. New Ruby options keep the existing defaults unless enabled.
|
|
37
|
+
|
|
38
|
+
- Two-phase checkpoints: `begin_checkpoint` returns a handle with `id` and `wait`.
|
|
39
|
+
Both checkpoint methods accept `scope: :all` to capture pending writes, or
|
|
40
|
+
`scope: :durable` (the existing default) for writes already durable.
|
|
41
|
+
- Point reads accept `max_concurrent_sources:` on databases, readers, snapshots,
|
|
42
|
+
and transactions; database metadata reads accept it too.
|
|
43
|
+
- `Database.open` accepts `experimental_incremental_compaction:` and
|
|
44
|
+
`sst_block_alignment:`. Both are disabled by default; see the warnings below.
|
|
45
|
+
- Snapshot and transaction range/prefix scans now accept `order:`, like database
|
|
46
|
+
and reader scans. Reader `merge_operator:` is now applied rather than ignored.
|
|
47
|
+
- **Breaking:** Ruby Proc/lambda merge operators are no longer accepted. They
|
|
48
|
+
silently used concatenation on other threads, which could change stored values.
|
|
49
|
+
Keep `:concat` / `:string_concat` for concatenation. For custom updates, use a
|
|
50
|
+
serializable transaction that reads the value and writes the computed result;
|
|
51
|
+
handle `SlateDb::TransactionError` conflicts in the application. Databases with
|
|
52
|
+
unresolved custom merge operands need an application-specific migration before
|
|
53
|
+
upgrading. Do not reopen them with `:concat` unless concatenation was their
|
|
54
|
+
intended merge rule.
|
|
55
|
+
- Reader disk-cache keywords remain supported, but upstream deprecated the
|
|
56
|
+
underlying `CachedObjectStore` in 0.18.1. The new mirror and Rust trait-based
|
|
57
|
+
extension APIs are not exposed by this gem yet.
|
|
58
|
+
|
|
59
|
+
Upstream also fixes clone-compaction retention/data-loss cases, writer ownership
|
|
60
|
+
validation after WAL replay, write stalls, upload retry/concurrency behavior, and
|
|
61
|
+
iterator fetch cancellation. It improves memtable seeks and point-read/scan work.
|
|
62
|
+
See the full [0.18.0 release notes](https://github.com/slatedb/slatedb/releases/tag/v0.18.0)
|
|
63
|
+
and [0.18.1 release notes](https://github.com/slatedb/slatedb/releases/tag/v0.18.1).
|
|
64
|
+
|
|
33
65
|
## Usage
|
|
34
66
|
|
|
35
67
|
### Basic Operations
|
|
@@ -64,6 +96,73 @@ SlateDb::Database.open("/tmp/mydb") do |db|
|
|
|
64
96
|
end # automatically closed
|
|
65
97
|
```
|
|
66
98
|
|
|
99
|
+
### Point-Read Concurrency
|
|
100
|
+
|
|
101
|
+
```ruby
|
|
102
|
+
db.get("key", max_concurrent_sources: 4, cache_blocks: false)
|
|
103
|
+
db.get_entry("key", max_concurrent_sources: 4) # value plus sequence/timestamps
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`max_concurrent_sources` defaults to 1 (serial fetching); zero also means 1.
|
|
107
|
+
SlateDB checks the newest source alone, then fetches up to this many sources
|
|
108
|
+
concurrently after a miss. Higher values can reduce latency but increase object
|
|
109
|
+
store requests and bandwidth. The same keyword works with `get` on readers,
|
|
110
|
+
snapshots, and transactions. It does not change isolation or durability.
|
|
111
|
+
|
|
112
|
+
### Compaction Controls
|
|
113
|
+
|
|
114
|
+
SlateDB 0.17 adds controls for how long compaction inputs remain protected from
|
|
115
|
+
garbage collection and when to consolidate sorted runs:
|
|
116
|
+
|
|
117
|
+
```ruby
|
|
118
|
+
SlateDb::Database.open("mydb",
|
|
119
|
+
url: "s3://mybucket/path",
|
|
120
|
+
compactor_checkpoint_lifetime: 30 * 60_000,
|
|
121
|
+
sorted_run_consolidation_threshold: 8) do |db|
|
|
122
|
+
db.put("key", "value")
|
|
123
|
+
end
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
`compactor_checkpoint_lifetime` is in milliseconds and defaults to 15 minutes.
|
|
127
|
+
Reads, scans, snapshots, and transactions using old compaction inputs must finish
|
|
128
|
+
within that time; after it expires, garbage collection can remove those inputs.
|
|
129
|
+
|
|
130
|
+
`sorted_run_consolidation_threshold` starts fallback consolidation when the
|
|
131
|
+
projected sorted-run count exceeds the threshold. It is not a hard limit.
|
|
132
|
+
Zero disables it (the default); use at least two to enable it. SlateDB warns and
|
|
133
|
+
disables the trigger for one. Low thresholds can increase storage reads and writes.
|
|
134
|
+
|
|
135
|
+
SlateDB 0.18 adds an opt-in experimental mode:
|
|
136
|
+
|
|
137
|
+
```ruby
|
|
138
|
+
SlateDb::Database.open("mydb", url: "s3://mybucket/path",
|
|
139
|
+
experimental_incremental_compaction: true) do |db|
|
|
140
|
+
db.put("key", "value")
|
|
141
|
+
end
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
This publishes output from sorted-run compactions at most once a minute before
|
|
145
|
+
the full compaction finishes. It can reduce read amplification and allow earlier
|
|
146
|
+
storage reclamation after the compactor checkpoint expires. Compactions with L0
|
|
147
|
+
inputs still publish only when complete. It is disabled by default, experimental,
|
|
148
|
+
and may change or disappear upstream.
|
|
149
|
+
|
|
150
|
+
### SST Block Alignment
|
|
151
|
+
|
|
152
|
+
`Database.open(..., sst_block_alignment: true)` pads SST data blocks to 4 KiB
|
|
153
|
+
boundaries, including oversized entries. It is disabled by default, uses more
|
|
154
|
+
storage, and cannot be combined with compression.
|
|
155
|
+
|
|
156
|
+
**Upgrade every reader before enabling alignment on a writer.** Older engines
|
|
157
|
+
that do not understand padded SSTs cannot read them. Once padded files exist,
|
|
158
|
+
rolling those readers back is not safe; disabling the option does not rewrite
|
|
159
|
+
existing files.
|
|
160
|
+
|
|
161
|
+
### Key Size
|
|
162
|
+
|
|
163
|
+
Keys can now exceed the previous 65,535-byte limit, up to SlateDB's new
|
|
164
|
+
4,294,967,295-byte limit. Very large keys still require enough memory to process.
|
|
165
|
+
|
|
67
166
|
### Persistent Storage
|
|
68
167
|
|
|
69
168
|
For persistent storage, provide an object store URL:
|
|
@@ -313,53 +412,19 @@ db.batch do |b|
|
|
|
313
412
|
end
|
|
314
413
|
```
|
|
315
414
|
|
|
316
|
-
####
|
|
317
|
-
|
|
318
|
-
You can provide a Ruby Proc/lambda as a custom merge operator:
|
|
319
|
-
|
|
320
|
-
```ruby
|
|
321
|
-
# Counter merge operator (adds numbers)
|
|
322
|
-
counter_merge = ->(key, existing, new_value) {
|
|
323
|
-
existing_num = existing ? existing.to_i : 0
|
|
324
|
-
(existing_num + new_value.to_i).to_s
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
SlateDb::Database.open("/tmp/mydb", merge_operator: counter_merge) do |db|
|
|
328
|
-
db.merge("visits", "1")
|
|
329
|
-
db.merge("visits", "1")
|
|
330
|
-
db.merge("visits", "1")
|
|
331
|
-
|
|
332
|
-
db.get("visits") # => "3"
|
|
333
|
-
end
|
|
415
|
+
#### Available Merge Operators
|
|
334
416
|
|
|
335
|
-
|
|
336
|
-
max_merge = ->(key, existing, new_value) {
|
|
337
|
-
existing_num = existing ? existing.to_i : 0
|
|
338
|
-
new_num = new_value.to_i
|
|
339
|
-
[existing_num, new_num].max.to_s
|
|
340
|
-
}
|
|
417
|
+
- `:string_concat` (or `:concat`) - Concatenates byte values (built-in)
|
|
341
418
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
db.merge("high_score", "250")
|
|
345
|
-
db.merge("high_score", "150")
|
|
419
|
+
Ruby Proc/lambda operators are rejected. Use a serializable read/modify/write
|
|
420
|
+
transaction for custom logic, with application-level conflict handling:
|
|
346
421
|
|
|
347
|
-
|
|
422
|
+
```ruby
|
|
423
|
+
db.transaction(isolation: :serializable) do |txn|
|
|
424
|
+
txn.put("visits", (txn.get("visits").to_i + 1).to_s)
|
|
348
425
|
end
|
|
349
426
|
```
|
|
350
427
|
|
|
351
|
-
The proc receives three arguments:
|
|
352
|
-
- `key` - The key being merged
|
|
353
|
-
- `existing` - The existing value (nil if no value exists)
|
|
354
|
-
- `new_value` - The new merge operand
|
|
355
|
-
|
|
356
|
-
**Note:** Custom Proc merge operators work best with direct `db.merge()` calls. When used with transactions or batches, some merge operations may be processed on background threads and fall back to string concatenation.
|
|
357
|
-
|
|
358
|
-
#### Available Merge Operators
|
|
359
|
-
|
|
360
|
-
- `:string_concat` (or `:concat`) - Concatenates byte values (built-in)
|
|
361
|
-
- Any `Proc` or `lambda` - Custom merge logic
|
|
362
|
-
|
|
363
428
|
### Write Batches
|
|
364
429
|
|
|
365
430
|
Perform multiple writes atomically:
|
|
@@ -467,6 +532,29 @@ SlateDb::Database.open("/tmp/mydb", url: "file:///tmp/mydb") do |db|
|
|
|
467
532
|
end
|
|
468
533
|
```
|
|
469
534
|
|
|
535
|
+
`create_checkpoint` waits until the checkpoint is stored. Its default
|
|
536
|
+
`scope: :durable` preserves the existing behavior. To include writes made with
|
|
537
|
+
`await_durable: false`, use `scope: :all`; no separate `flush` is needed.
|
|
538
|
+
|
|
539
|
+
To separate capturing the write boundary from waiting for storage:
|
|
540
|
+
|
|
541
|
+
```ruby
|
|
542
|
+
db.put("key", "before", await_durable: false)
|
|
543
|
+
handle = db.begin_checkpoint(scope: :all, name: "backup", lifetime: 3_600_000)
|
|
544
|
+
id = handle.id # UUID is available immediately
|
|
545
|
+
db.put("key", "after") # not part of this checkpoint
|
|
546
|
+
checkpoint = handle.wait # => { id: id, manifest_id: ... }
|
|
547
|
+
```
|
|
548
|
+
|
|
549
|
+
With `scope: :all`, writes completed before `begin_checkpoint` starts are included;
|
|
550
|
+
writes started after it returns are excluded. Concurrent writes during the call
|
|
551
|
+
may be included. Starting the checkpoint can still wait for internal flush work;
|
|
552
|
+
it is not a fully nonblocking Ruby call. `wait` releases the Ruby GVL while
|
|
553
|
+
waiting, consumes the handle, and raises `SlateDb::ClosedError` if called again.
|
|
554
|
+
`id` remains readable. Dropping the handle does not cancel or delete the checkpoint.
|
|
555
|
+
If waiting fails, use the UUID with `Admin#list_checkpoints` to check its outcome
|
|
556
|
+
before starting another checkpoint.
|
|
557
|
+
|
|
470
558
|
### Snapshots
|
|
471
559
|
|
|
472
560
|
Point-in-time consistent reads:
|
|
@@ -489,6 +577,11 @@ value = snap.get("key")
|
|
|
489
577
|
snap.close
|
|
490
578
|
```
|
|
491
579
|
|
|
580
|
+
Range and prefix scans on snapshots and transactions accept the same `order:`
|
|
581
|
+
keyword as databases and readers, including `:asc`/`:ascending` and
|
|
582
|
+
`:desc`/`:descending`. Bounds remain lower-inclusive and upper-exclusive in
|
|
583
|
+
either direction.
|
|
584
|
+
|
|
492
585
|
### Reader (Read-Only Access)
|
|
493
586
|
|
|
494
587
|
Open a database in read-only mode, useful for replicas:
|
|
@@ -501,6 +594,14 @@ SlateDb::Reader.open("/tmp/mydb", url: "s3://bucket/path") do |reader|
|
|
|
501
594
|
reader.scan("prefix:").each do |k, v|
|
|
502
595
|
puts "#{k}: #{v}"
|
|
503
596
|
end
|
|
597
|
+
|
|
598
|
+
# Range and prefix scans accept :asc/:ascending or :desc/:descending.
|
|
599
|
+
reader.scan("user:100", "user:200", order: :desc).each do |k, v|
|
|
600
|
+
puts "#{k}: #{v}"
|
|
601
|
+
end
|
|
602
|
+
reader.scan_prefix("user:", from: "100", to: "200", order: :desc).each do |k, v|
|
|
603
|
+
puts "#{k}: #{v}"
|
|
604
|
+
end
|
|
504
605
|
end
|
|
505
606
|
|
|
506
607
|
# Open at a specific checkpoint (pins the reader to that checkpoint's state)
|
|
@@ -532,6 +633,14 @@ SlateDb::Reader.open("/tmp/mydb",
|
|
|
532
633
|
end
|
|
533
634
|
```
|
|
534
635
|
|
|
636
|
+
When the writer uses `merge_operator: :concat`, pass the same operator to
|
|
637
|
+
`Reader.open` so WAL replay and reads apply the correct merge rule. Unknown
|
|
638
|
+
operator names now raise `SlateDb::InvalidArgumentError` instead of being ignored.
|
|
639
|
+
|
|
640
|
+
The `cache_root` and `max_open_file_handles` keywords still work, but SlateDB
|
|
641
|
+
0.18.1 deprecates this legacy disk cache in favor of its separate mirror crate or
|
|
642
|
+
tiered block cache. Those replacement APIs are not yet exposed here.
|
|
643
|
+
|
|
535
644
|
### Admin Operations
|
|
536
645
|
|
|
537
646
|
Administrative operations for database management:
|
|
@@ -558,9 +667,6 @@ admin.delete_checkpoint("uuid")
|
|
|
558
667
|
# Garbage Collection
|
|
559
668
|
admin.run_gc # Run with default settings
|
|
560
669
|
admin.run_gc(min_age: 3600_000) # Set min age for all directories (1 hour)
|
|
561
|
-
admin.run_gc(manifest_min_age: 86400_000) # Custom age for manifest (1 day)
|
|
562
|
-
admin.run_gc(wal_min_age: 60_000) # Custom age for WAL (1 minute)
|
|
563
|
-
admin.run_gc(compacted_min_age: 60_000) # Custom age for compacted (1 minute)
|
|
564
670
|
|
|
565
671
|
# Delete a database (SlateDB >= 0.16.0)
|
|
566
672
|
would_delete = admin.delete_db # Dry run: returns the object paths
|
data/ext/slatedb/Cargo.toml
CHANGED
|
@@ -11,13 +11,12 @@ name = "slatedb"
|
|
|
11
11
|
crate-type = ["cdylib"]
|
|
12
12
|
|
|
13
13
|
[dependencies]
|
|
14
|
-
slatedb = "0.
|
|
15
|
-
magnus = { version = "0.
|
|
16
|
-
rb-sys = { version = "0.9.
|
|
17
|
-
tokio = { version = "1.
|
|
18
|
-
bytes = "1.
|
|
19
|
-
serde_json = "1.0.
|
|
14
|
+
slatedb = "0.18.1"
|
|
15
|
+
magnus = { version = "0.9.3", features = ["rb-sys"] }
|
|
16
|
+
rb-sys = { version = "0.9.130", features = ["stable-api-compiled-fallback"] }
|
|
17
|
+
tokio = { version = "1.54.1", features = ["rt-multi-thread", "sync"] }
|
|
18
|
+
bytes = "1.12.1"
|
|
19
|
+
serde_json = "1.0.154"
|
|
20
20
|
url = "2.5.8"
|
|
21
21
|
once_cell = "1.21.4"
|
|
22
|
-
|
|
23
|
-
uuid = "1.23.1"
|
|
22
|
+
uuid = "1.29.2"
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
use magnus::prelude::*;
|
|
2
|
+
use magnus::{method, Error, RHash, Ruby};
|
|
3
|
+
use std::cell::RefCell;
|
|
4
|
+
|
|
5
|
+
use crate::errors::closed_error;
|
|
6
|
+
use crate::runtime::block_on_result;
|
|
7
|
+
|
|
8
|
+
#[magnus::wrap(class = "SlateDb::CheckpointHandle", free_immediately, size)]
|
|
9
|
+
pub struct CheckpointHandle {
|
|
10
|
+
id: String,
|
|
11
|
+
inner: RefCell<Option<slatedb::CheckpointHandle>>,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
impl CheckpointHandle {
|
|
15
|
+
pub fn new(handle: slatedb::CheckpointHandle) -> Self {
|
|
16
|
+
Self {
|
|
17
|
+
id: handle.id().to_string(),
|
|
18
|
+
inner: RefCell::new(Some(handle)),
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
pub fn id(&self) -> String {
|
|
23
|
+
self.id.clone()
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
pub fn wait(&self) -> Result<RHash, Error> {
|
|
27
|
+
let handle = self
|
|
28
|
+
.inner
|
|
29
|
+
.borrow_mut()
|
|
30
|
+
.take()
|
|
31
|
+
.ok_or_else(|| closed_error("checkpoint handle has already been waited on"))?;
|
|
32
|
+
let result = block_on_result(handle.wait())?;
|
|
33
|
+
|
|
34
|
+
let ruby = Ruby::get().expect("Ruby runtime not available");
|
|
35
|
+
let hash = ruby.hash_new();
|
|
36
|
+
hash.aset(ruby.to_symbol("id"), result.id.to_string())?;
|
|
37
|
+
hash.aset(ruby.to_symbol("manifest_id"), result.manifest_id)?;
|
|
38
|
+
Ok(hash)
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
pub fn define_checkpoint_class(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
43
|
+
let class = module.define_class("CheckpointHandle", ruby.class_object())?;
|
|
44
|
+
class.define_method("id", method!(CheckpointHandle::id, 0))?;
|
|
45
|
+
class.define_method("wait", method!(CheckpointHandle::wait, 0))?;
|
|
46
|
+
Ok(())
|
|
47
|
+
}
|
data/ext/slatedb/src/database.rs
CHANGED
|
@@ -5,14 +5,15 @@ use magnus::prelude::*;
|
|
|
5
5
|
use magnus::{function, method, Error, RHash, Ruby};
|
|
6
6
|
use slatedb::config::{
|
|
7
7
|
CloseOptions, DurabilityLevel, FlushOptions, FlushType, MergeOptions, PutOptions, ReadOptions,
|
|
8
|
-
ScanOptions, Ttl, WriteOptions,
|
|
8
|
+
ScanOptions, Settings, Ttl, WriteOptions,
|
|
9
9
|
};
|
|
10
10
|
use slatedb::object_store::memory::InMemory;
|
|
11
11
|
use slatedb::{Db, IsolationLevel, IterationOrder, KeyValue};
|
|
12
12
|
|
|
13
|
+
use crate::checkpoint::CheckpointHandle;
|
|
13
14
|
use crate::errors::invalid_argument_error;
|
|
14
15
|
use crate::iterator::Iterator;
|
|
15
|
-
use crate::merge_ops::
|
|
16
|
+
use crate::merge_ops::parse_merge_operator;
|
|
16
17
|
use crate::metrics::Metrics;
|
|
17
18
|
use crate::runtime::block_on_result;
|
|
18
19
|
use crate::snapshot::Snapshot;
|
|
@@ -76,6 +77,10 @@ impl Database {
|
|
|
76
77
|
opts.cache_blocks = cb;
|
|
77
78
|
}
|
|
78
79
|
|
|
80
|
+
if let Some(limit) = get_optional::<usize>(kwargs, "max_concurrent_sources")? {
|
|
81
|
+
opts.max_concurrent_sources = limit;
|
|
82
|
+
}
|
|
83
|
+
|
|
79
84
|
Ok(opts)
|
|
80
85
|
}
|
|
81
86
|
|
|
@@ -113,13 +118,35 @@ impl Database {
|
|
|
113
118
|
/// # Arguments
|
|
114
119
|
/// * `path` - The path identifier for the database
|
|
115
120
|
/// * `url` - Optional object store URL (e.g., "s3://bucket/path")
|
|
116
|
-
/// * `kwargs` - Additional options (merge_operator
|
|
121
|
+
/// * `kwargs` - Additional options (merge_operator and compaction settings)
|
|
117
122
|
///
|
|
118
123
|
/// # Returns
|
|
119
124
|
/// A new Database instance
|
|
120
125
|
pub fn open(path: String, url: Option<String>, kwargs: RHash) -> Result<Self, Error> {
|
|
121
|
-
|
|
122
|
-
let
|
|
126
|
+
let merge_operator = parse_merge_operator(&kwargs)?;
|
|
127
|
+
let checkpoint_lifetime = get_optional::<u64>(&kwargs, "compactor_checkpoint_lifetime")?
|
|
128
|
+
.map(std::time::Duration::from_millis);
|
|
129
|
+
let consolidation_threshold =
|
|
130
|
+
get_optional::<usize>(&kwargs, "sorted_run_consolidation_threshold")?;
|
|
131
|
+
let incremental_compaction =
|
|
132
|
+
get_optional::<bool>(&kwargs, "experimental_incremental_compaction")?;
|
|
133
|
+
let block_alignment = get_optional::<bool>(&kwargs, "sst_block_alignment")?;
|
|
134
|
+
|
|
135
|
+
let mut settings = Settings::default();
|
|
136
|
+
if let Some(compactor) = settings.compactor_options.as_mut() {
|
|
137
|
+
if let Some(enabled) = incremental_compaction {
|
|
138
|
+
compactor.enable_experimental_incremental_compaction = enabled;
|
|
139
|
+
}
|
|
140
|
+
if let Some(lifetime) = checkpoint_lifetime {
|
|
141
|
+
compactor.checkpoint_lifetime = lifetime;
|
|
142
|
+
}
|
|
143
|
+
if let Some(threshold) = consolidation_threshold {
|
|
144
|
+
compactor.scheduler_options.insert(
|
|
145
|
+
"sorted_run_consolidation_threshold".to_string(),
|
|
146
|
+
threshold.to_string(),
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
123
150
|
|
|
124
151
|
let db = block_on_result(async {
|
|
125
152
|
let object_store: Arc<dyn slatedb::object_store::ObjectStore> =
|
|
@@ -129,7 +156,10 @@ impl Database {
|
|
|
129
156
|
Arc::new(InMemory::new())
|
|
130
157
|
};
|
|
131
158
|
|
|
132
|
-
let mut builder = Db::builder(path, object_store);
|
|
159
|
+
let mut builder = Db::builder(path, object_store).with_settings(settings);
|
|
160
|
+
if let Some(aligned) = block_alignment {
|
|
161
|
+
builder = builder.with_sst_block_alignment(aligned);
|
|
162
|
+
}
|
|
133
163
|
if let Some(merge_operator) = merge_operator {
|
|
134
164
|
builder = builder.with_merge_operator(merge_operator);
|
|
135
165
|
}
|
|
@@ -696,7 +726,10 @@ impl Database {
|
|
|
696
726
|
}
|
|
697
727
|
};
|
|
698
728
|
|
|
699
|
-
let txn =
|
|
729
|
+
let txn = self
|
|
730
|
+
.inner
|
|
731
|
+
.begin(isolation_level)
|
|
732
|
+
.map_err(crate::errors::map_error)?;
|
|
700
733
|
Ok(Transaction::new(txn))
|
|
701
734
|
}
|
|
702
735
|
|
|
@@ -705,7 +738,7 @@ impl Database {
|
|
|
705
738
|
/// # Returns
|
|
706
739
|
/// A new Snapshot instance
|
|
707
740
|
pub fn snapshot(&self) -> Result<Snapshot, Error> {
|
|
708
|
-
let snap =
|
|
741
|
+
let snap = self.inner.snapshot().map_err(crate::errors::map_error)?;
|
|
709
742
|
Ok(Snapshot::new(snap))
|
|
710
743
|
}
|
|
711
744
|
|
|
@@ -717,8 +750,25 @@ impl Database {
|
|
|
717
750
|
/// # Returns
|
|
718
751
|
/// Hash with id (UUID string) and manifest_id (int)
|
|
719
752
|
pub fn create_checkpoint(&self, kwargs: RHash) -> Result<RHash, Error> {
|
|
753
|
+
self.begin_checkpoint(kwargs)?.wait()
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
pub fn begin_checkpoint(&self, kwargs: RHash) -> Result<CheckpointHandle, Error> {
|
|
720
757
|
use slatedb::config::{CheckpointOptions, CheckpointScope};
|
|
721
758
|
|
|
759
|
+
let scope = match get_optional::<String>(&kwargs, "scope")?
|
|
760
|
+
.as_deref()
|
|
761
|
+
.unwrap_or("durable")
|
|
762
|
+
{
|
|
763
|
+
"durable" => CheckpointScope::Durable,
|
|
764
|
+
"all" => CheckpointScope::All,
|
|
765
|
+
other => {
|
|
766
|
+
return Err(invalid_argument_error(&format!(
|
|
767
|
+
"invalid scope: {} (expected 'durable' or 'all')",
|
|
768
|
+
other
|
|
769
|
+
)))
|
|
770
|
+
}
|
|
771
|
+
};
|
|
722
772
|
let lifetime =
|
|
723
773
|
get_optional::<u64>(&kwargs, "lifetime")?.map(std::time::Duration::from_millis);
|
|
724
774
|
let name = get_optional::<String>(&kwargs, "name")?;
|
|
@@ -729,18 +779,9 @@ impl Database {
|
|
|
729
779
|
name,
|
|
730
780
|
};
|
|
731
781
|
|
|
732
|
-
let result = block_on_result(async {
|
|
733
|
-
self.inner
|
|
734
|
-
.create_checkpoint(CheckpointScope::Durable, &options)
|
|
735
|
-
.await
|
|
736
|
-
})?;
|
|
782
|
+
let result = block_on_result(async { self.inner.begin_checkpoint(scope, &options).await })?;
|
|
737
783
|
|
|
738
|
-
|
|
739
|
-
let hash = ruby.hash_new();
|
|
740
|
-
hash.aset(ruby.to_symbol("id"), result.id.to_string())?;
|
|
741
|
-
hash.aset(ruby.to_symbol("manifest_id"), result.manifest_id)?;
|
|
742
|
-
|
|
743
|
-
Ok(hash)
|
|
784
|
+
Ok(CheckpointHandle::new(result))
|
|
744
785
|
}
|
|
745
786
|
|
|
746
787
|
/// Flush the database to ensure durability.
|
|
@@ -842,6 +883,7 @@ pub fn define_database_class(ruby: &Ruby, module: &magnus::RModule) -> Result<()
|
|
|
842
883
|
"_create_checkpoint",
|
|
843
884
|
method!(Database::create_checkpoint, 1),
|
|
844
885
|
)?;
|
|
886
|
+
class.define_method("_begin_checkpoint", method!(Database::begin_checkpoint, 1))?;
|
|
845
887
|
class.define_method("_flush", method!(Database::flush, 0))?;
|
|
846
888
|
class.define_method(
|
|
847
889
|
"_flush_with_options",
|
data/ext/slatedb/src/lib.rs
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
use magnus::{Error, Ruby};
|
|
18
18
|
|
|
19
19
|
mod admin;
|
|
20
|
+
mod checkpoint;
|
|
20
21
|
mod database;
|
|
21
22
|
mod errors;
|
|
22
23
|
mod iterator;
|
|
@@ -40,6 +41,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
40
41
|
errors::define_exceptions(ruby, &module)?;
|
|
41
42
|
|
|
42
43
|
// Define core classes
|
|
44
|
+
checkpoint::define_checkpoint_class(ruby, &module)?;
|
|
43
45
|
database::define_database_class(ruby, &module)?;
|
|
44
46
|
iterator::define_iterator_class(ruby, &module)?;
|
|
45
47
|
write_batch::define_write_batch_class(ruby, &module)?;
|