slatedb 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c93234ef6d9251d54b90cd816983c1d073d159587ae655fbef8e4e3b433928a3
-  data.tar.gz: 0b7aa7b9137ff2228f7e4153da2daafbb235ce857b51faa8d552c55f18cb7e7a
+  metadata.gz: 0026c9e2a5afd6e0d7fa35c10b037791910a2438cf0bcec73c9935db557c9a85
+  data.tar.gz: 140bdf13e24d576e87165a63f31e3a0130244f36ea1278da7da5e6703347f7c5
 SHA512:
-  metadata.gz: 54cc3a48576fc4ca07dc6ae95a755aa6fc5dd7fa7af46eba3452f3465c74d005d9f29d06ac1cc9ddf3461d602d910d63a08f55b0d486140ed22024f3ce1e3e4e
-  data.tar.gz: 2bd1c37ca9fcfa737bd47fd3d76b136376819b573a08e044c06114f5555d5243e8b465b7bbc8cba02f3461982cc5ea1e4195fb31e53a5f0d4dcd608d1ea79f55
+  metadata.gz: ae8fe00e7069801bfdf8cf90ea48bfee6a60ed1413cdbc8bef5b9a083744436a0b3d3c1e46ef51c5cbd5de85bd46c6c332ec4b016ffd6482a729fa0b3a50e531
+  data.tar.gz: 5e60f6509c2eae4c948751d941f6cd7389dae8e43e241d02d559c47251b26383c6a59fcedb50133d54759d9d791a9f07079f6eed37106a386358094f2f3a5b7e

data/README.md

@@ -181,6 +181,105 @@ users = db.scan("user:").select { |k, v| v.include?("active") }
 all_entries = db.scan("").to_a
 ```
 
+#### Prefix Scanning
+
+Scan all keys with a given prefix using `scan_prefix`:
+
+```ruby
+# Scan all keys starting with "user:"
+db.scan_prefix("user:").each do |key, value|
+  puts "#{key}: #{value}"
+end
+
+# Block form
+db.scan_prefix("order:") do |key, value|
+  puts "#{key}: #{value}"
+end
+
+# Works with transactions, snapshots, and readers too
+db.transaction do |txn|
+  txn.scan_prefix("item:").each do |k, v|
+    puts "#{k}: #{v}"
+  end
+end
+```
+
+### Merge Operations
+
+Merge operations allow you to combine values without reading them first, useful for counters, append-only logs, and similar patterns:
+
+```ruby
+# Open with a built-in merge operator
+SlateDb::Database.open("/tmp/mydb", merge_operator: :string_concat) do |db|
+  # Merge appends to existing values (or creates if key doesn't exist)
+  db.merge("log", "line1\n")
+  db.merge("log", "line2\n")
+  db.merge("log", "line3\n")
+
+  db.get("log")  # => "line1\nline2\nline3\n"
+end
+
+# Merge with options
+db.merge("key", "value", ttl: 60_000, await_durable: false)
+
+# Works in transactions and batches
+db.transaction do |txn|
+  txn.merge("counter", "1")
+end
+
+db.batch do |b|
+  b.merge("key", "a")
+   .merge("key", "b")
+end
+```
+
+#### Custom Merge Operators
+
+You can provide a Ruby Proc/lambda as a custom merge operator:
+
+```ruby
+# Counter merge operator (adds numbers)
+counter_merge = ->(key, existing, new_value) {
+  existing_num = existing ? existing.to_i : 0
+  (existing_num + new_value.to_i).to_s
+}
+
+SlateDb::Database.open("/tmp/mydb", merge_operator: counter_merge) do |db|
+  db.merge("visits", "1")
+  db.merge("visits", "1")
+  db.merge("visits", "1")
+
+  db.get("visits")  # => "3"
+end
+
+# Max value merge operator
+max_merge = ->(key, existing, new_value) {
+  existing_num = existing ? existing.to_i : 0
+  new_num = new_value.to_i
+  [existing_num, new_num].max.to_s
+}
+
+SlateDb::Database.open("/tmp/mydb", merge_operator: max_merge) do |db|
+  db.merge("high_score", "100")
+  db.merge("high_score", "250")
+  db.merge("high_score", "150")
+
+  db.get("high_score")  # => "250"
+end
+```
+
+The proc receives three arguments:
+- `key` - The key being merged
+- `existing` - The existing value (nil if no value exists)
+- `new_value` - The new merge operand
+
+**Note:** Custom Proc merge operators work best with direct `db.merge()` calls. When used with transactions or batches, some merge operations may be processed on background threads and fall back to string concatenation.
+
+#### Available Merge Operators
+
+- `:string_concat` (or `:concat`) - Concatenates byte values (built-in)
+- Any `Proc` or `lambda` - Custom merge logic
+
 ### Write Batches
 
 Perform multiple writes atomically:
@@ -231,18 +330,60 @@ Transaction operations:
 db.transaction do |txn|
   # Read
   value = txn.get("key")
-  
+
   # Write
   txn.put("key", "value")
   txn.put("expiring", "data", ttl: 30_000)
-  
+
   # Delete
   txn.delete("old_key")
-  
+
   # Scan
   txn.scan("prefix:").each do |k, v|
     puts "#{k}: #{v}"
   end
+
+  # Scan with prefix
+  txn.scan_prefix("user:").each do |k, v|
+    puts "#{k}: #{v}"
+  end
+end
+```
+
+#### Explicit Read Tracking
+
+In serializable transactions, use `mark_read` to explicitly track keys for conflict detection without actually reading them:
+
+```ruby
+db.transaction(isolation: :serializable) do |txn|
+  # Mark keys as read for conflict detection
+  txn.mark_read(["key1", "key2", "key3"])
+
+  # Now if another transaction modifies key1/key2/key3,
+  # this transaction will fail on commit
+  txn.put("result", "computed_value")
+end
+```
+
+### Checkpoints
+
+Create durable checkpoints for backup or read replica purposes:
+
+```ruby
+SlateDb::Database.open("/tmp/mydb", url: "file:///tmp/mydb") do |db|
+  db.put("key", "value")
+  db.flush
+
+  # Create a checkpoint
+  checkpoint = db.create_checkpoint
+  puts "Checkpoint ID: #{checkpoint[:id]}"
+  puts "Manifest ID: #{checkpoint[:manifest_id]}"
+
+  # Create a named checkpoint with lifetime
+  checkpoint = db.create_checkpoint(
+    name: "before-migration",
+    lifetime: 3_600_000  # 1 hour in milliseconds
+  )
 end
 ```
 

data/ext/slatedb/Cargo.toml

@@ -11,13 +11,12 @@ name = "slatedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-slatedb = "0.9"
-magnus = { version = "0.8", features = ["rb-sys"] }
-rb-sys = { version = "0.9", features = ["stable-api-compiled-fallback"] }
-tokio = { version = "1", features = ["rt-multi-thread", "sync"] }
-bytes = "1"
-object_store = { version = "0.12", features = ["aws"] }
-url = "2"
-once_cell = "1"
-log = "0.4"
-uuid = "1"
+slatedb = "0.10"
+magnus = { version = "0.8.2", features = ["rb-sys"] }
+rb-sys = { version = "0.9.123", features = ["stable-api-compiled-fallback"] }
+tokio = { version = "1.47.2", features = ["rt-multi-thread", "sync"] }
+bytes = "1.11.0"
+url = "2.5.7"
+once_cell = "1.21.3"
+log = "0.4.29"
+uuid = "1.19.0"

data/ext/slatedb/src/admin.rs

@@ -25,10 +25,10 @@ impl Admin {
     /// * `path` - The path identifier for the database
     /// * `url` - Optional object store URL
     pub fn new(path: String, url: Option<String>) -> Result<Self, Error> {
-        let object_store: Arc<dyn object_store::ObjectStore> = if let Some(ref url) = url {
+        let object_store: Arc<dyn slatedb::object_store::ObjectStore> = if let Some(ref url) = url {
             block_on_result(async { resolve_object_store(url) })?
         } else {
-            Arc::new(object_store::memory::InMemory::new())
+            Arc::new(slatedb::object_store::memory::InMemory::new())
         };
 
         let admin = AdminBuilder::new(path, object_store).build();
@@ -235,6 +235,7 @@ impl Admin {
                     min_age,
                     default_opts.compacted_options,
                 ),
+                compactions_options: default_opts.compactions_options,
             }
         };
 

data/ext/slatedb/src/database.rs

@@ -2,12 +2,15 @@ use std::sync::Arc;
 
 use magnus::prelude::*;
 use magnus::{function, method, Error, RHash, Ruby};
-use slatedb::config::{DurabilityLevel, PutOptions, ReadOptions, ScanOptions, Ttl, WriteOptions};
+use slatedb::config::{
+    DurabilityLevel, MergeOptions, PutOptions, ReadOptions, ScanOptions, Ttl, WriteOptions,
+};
 use slatedb::object_store::memory::InMemory;
 use slatedb::{Db, IsolationLevel};
 
 use crate::errors::invalid_argument_error;
 use crate::iterator::Iterator;
+use crate::merge_ops::{parse_merge_operator, parse_merge_operator_proc};
 use crate::runtime::block_on_result;
 use crate::snapshot::Snapshot;
 use crate::transaction::Transaction;
@@ -28,18 +31,29 @@ impl Database {
     /// # Arguments
     /// * `path` - The path identifier for the database
     /// * `url` - Optional object store URL (e.g., "s3://bucket/path")
+    /// * `kwargs` - Additional options (merge_operator, merge_operator_proc)
     ///
     /// # Returns
     /// A new Database instance
-    pub fn open(path: String, url: Option<String>) -> Result<Self, Error> {
+    pub fn open(path: String, url: Option<String>, kwargs: RHash) -> Result<Self, Error> {
+        // Try string-based merge operator first, then proc-based
+        let merge_operator = parse_merge_operator(&kwargs)?
+            .or(parse_merge_operator_proc(&kwargs)?);
+
         let db = block_on_result(async {
-            let object_store: Arc<dyn object_store::ObjectStore> = if let Some(ref url_str) = url {
-                resolve_object_store(url_str)?
-            } else {
-                Arc::new(InMemory::new())
-            };
+            let object_store: Arc<dyn slatedb::object_store::ObjectStore> =
+                if let Some(ref url_str) = url {
+                    resolve_object_store(url_str)?
+                } else {
+                    Arc::new(InMemory::new())
+                };
+
+            let mut builder = Db::builder(path, object_store);
+            if let Some(merge_operator) = merge_operator {
+                builder = builder.with_merge_operator(merge_operator);
+            }
 
-            Db::builder(path, object_store).build().await
+            builder.build().await
         })?;
 
         Ok(Self {
@@ -325,6 +339,85 @@ impl Database {
         Ok(Iterator::new(iter))
     }
 
+    /// Scan all keys with a given prefix.
+    ///
+    /// # Arguments
+    /// * `prefix` - The key prefix to scan
+    ///
+    /// # Returns
+    /// An Iterator over key-value pairs
+    pub fn scan_prefix(&self, prefix: String) -> Result<Iterator, Error> {
+        if prefix.is_empty() {
+            return Err(invalid_argument_error("prefix cannot be empty"));
+        }
+
+        let opts = ScanOptions::default();
+        let iter = block_on_result(async {
+            self.inner
+                .scan_prefix_with_options(prefix.as_bytes(), &opts)
+                .await
+        })?;
+
+        Ok(Iterator::new(iter))
+    }
+
+    /// Scan all keys with a given prefix with options.
+    ///
+    /// # Arguments
+    /// * `prefix` - The key prefix to scan
+    /// * `kwargs` - Keyword arguments (durability_filter, dirty, read_ahead_bytes, cache_blocks, max_fetch_tasks)
+    ///
+    /// # Returns
+    /// An Iterator over key-value pairs
+    pub fn scan_prefix_with_options(
+        &self,
+        prefix: String,
+        kwargs: RHash,
+    ) -> Result<Iterator, Error> {
+        if prefix.is_empty() {
+            return Err(invalid_argument_error("prefix cannot be empty"));
+        }
+
+        let mut opts = ScanOptions::default();
+
+        if let Some(df) = get_optional::<String>(&kwargs, "durability_filter")? {
+            opts.durability_filter = match df.as_str() {
+                "remote" => DurabilityLevel::Remote,
+                "memory" => DurabilityLevel::Memory,
+                other => {
+                    return Err(invalid_argument_error(&format!(
+                        "invalid durability_filter: {} (expected 'remote' or 'memory')",
+                        other
+                    )))
+                }
+            };
+        }
+
+        if let Some(dirty) = get_optional::<bool>(&kwargs, "dirty")? {
+            opts.dirty = dirty;
+        }
+
+        if let Some(rab) = get_optional::<usize>(&kwargs, "read_ahead_bytes")? {
+            opts.read_ahead_bytes = rab;
+        }
+
+        if let Some(cb) = get_optional::<bool>(&kwargs, "cache_blocks")? {
+            opts.cache_blocks = cb;
+        }
+
+        if let Some(mft) = get_optional::<usize>(&kwargs, "max_fetch_tasks")? {
+            opts.max_fetch_tasks = mft;
+        }
+
+        let iter = block_on_result(async {
+            self.inner
+                .scan_prefix_with_options(prefix.as_bytes(), &opts)
+                .await
+        })?;
+
+        Ok(Iterator::new(iter))
+    }
+
     /// Write a batch of operations atomically.
     ///
     /// # Arguments
@@ -355,6 +448,62 @@ impl Database {
         Ok(())
     }
 
+    /// Merge a value into the database.
+    ///
+    /// # Arguments
+    /// * `key` - The key to merge into
+    /// * `value` - The merge operand to apply
+    pub fn merge(&self, key: String, value: String) -> Result<(), Error> {
+        if key.is_empty() {
+            return Err(invalid_argument_error("key cannot be empty"));
+        }
+
+        let merge_opts = MergeOptions { ttl: Ttl::Default };
+
+        let write_opts = WriteOptions {
+            await_durable: true,
+        };
+
+        block_on_result(async {
+            self.inner
+                .merge_with_options(key.as_bytes(), value.as_bytes(), &merge_opts, &write_opts)
+                .await
+        })?;
+
+        Ok(())
+    }
+
+    /// Merge a value into the database with options.
+    ///
+    /// # Arguments
+    /// * `key` - The key to merge into
+    /// * `value` - The merge operand to apply
+    /// * `kwargs` - Keyword arguments (ttl, await_durable)
+    pub fn merge_with_options(&self, key: String, value: String, kwargs: RHash) -> Result<(), Error> {
+        if key.is_empty() {
+            return Err(invalid_argument_error("key cannot be empty"));
+        }
+
+        let ttl = get_optional::<u64>(&kwargs, "ttl")?;
+        let merge_opts = MergeOptions {
+            ttl: match ttl {
+                Some(ms) => Ttl::ExpireAfter(ms),
+                None => Ttl::Default,
+            },
+        };
+
+        let await_durable = get_optional::<bool>(&kwargs, "await_durable")?.unwrap_or(true);
+        let write_opts = WriteOptions { await_durable };
+
+        block_on_result(async {
+            self.inner
+                .merge_with_options(key.as_bytes(), value.as_bytes(), &merge_opts, &write_opts)
+                .await
+        })?;
+
+        Ok(())
+    }
+
     /// Begin a new transaction.
     ///
     /// # Arguments
@@ -389,6 +538,40 @@ impl Database {
         Ok(Snapshot::new(snap))
     }
 
+    /// Create a checkpoint of the database.
+    ///
+    /// # Arguments
+    /// * `kwargs` - Options: lifetime (ms), name
+    ///
+    /// # Returns
+    /// Hash with id (UUID string) and manifest_id (int)
+    pub fn create_checkpoint(&self, kwargs: RHash) -> Result<RHash, Error> {
+        use slatedb::config::{CheckpointOptions, CheckpointScope};
+
+        let lifetime = get_optional::<u64>(&kwargs, "lifetime")?
+            .map(std::time::Duration::from_millis);
+        let name = get_optional::<String>(&kwargs, "name")?;
+
+        let options = CheckpointOptions {
+            lifetime,
+            source: None,
+            name,
+        };
+
+        let result = block_on_result(async {
+            self.inner
+                .create_checkpoint(CheckpointScope::Durable, &options)
+                .await
+        })?;
+
+        let ruby = Ruby::get().expect("Ruby runtime not available");
+        let hash = ruby.hash_new();
+        hash.aset(ruby.to_symbol("id"), result.id.to_string())?;
+        hash.aset(ruby.to_symbol("manifest_id"), result.manifest_id)?;
+
+        Ok(hash)
+    }
+
     /// Flush the database to ensure durability.
     pub fn flush(&self) -> Result<(), Error> {
         block_on_result(async { self.inner.flush().await })?;
@@ -407,7 +590,7 @@ pub fn define_database_class(ruby: &Ruby, module: &magnus::RModule) -> Result<()
     let class = module.define_class("Database", ruby.class_object())?;
 
     // Class methods
-    class.define_singleton_method("_open", function!(Database::open, 2))?;
+    class.define_singleton_method("_open", function!(Database::open, 3))?;
 
     // Instance methods - simple versions
     class.define_method("_get", method!(Database::get, 1))?;
@@ -425,16 +608,30 @@ pub fn define_database_class(ruby: &Ruby, module: &magnus::RModule) -> Result<()
         "_scan_with_options",
         method!(Database::scan_with_options, 3),
     )?;
+    class.define_method("_scan_prefix", method!(Database::scan_prefix, 1))?;
+    class.define_method(
+        "_scan_prefix_with_options",
+        method!(Database::scan_prefix_with_options, 2),
+    )?;
     class.define_method("_write", method!(Database::write, 1))?;
     class.define_method(
         "_write_with_options",
         method!(Database::write_with_options, 2),
     )?;
+    class.define_method("_merge", method!(Database::merge, 2))?;
+    class.define_method(
+        "_merge_with_options",
+        method!(Database::merge_with_options, 3),
+    )?;
     class.define_method(
         "_begin_transaction",
         method!(Database::begin_transaction, 1),
     )?;
     class.define_method("_snapshot", method!(Database::snapshot, 0))?;
+    class.define_method(
+        "_create_checkpoint",
+        method!(Database::create_checkpoint, 1),
+    )?;
     class.define_method("flush", method!(Database::flush, 0))?;
     class.define_method("close", method!(Database::close, 0))?;
 

data/ext/slatedb/src/lib.rs

@@ -20,6 +20,7 @@ mod admin;
 mod database;
 mod errors;
 mod iterator;
+mod merge_ops;
 mod reader;
 mod runtime;
 mod snapshot;