slatedb 0.5.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd1077018f7a8738545fc60b626a0e51d3240ae4141d9616787bf642489a22bc
-  data.tar.gz: 7aea1d1d0f9e1e3ee349d742246b7f89353ab8af39a6aca12d3167d7064d9023
+  metadata.gz: c46e1c81aa2f9f98834847d89ebe70a3caa18dd7307ec864d2d18fb448b8c366
+  data.tar.gz: 1e24f5bb9dc914c3f9f5fe7025d481b0eb93160ffd5834f78d039c333d66e434
 SHA512:
-  metadata.gz: 2fcbdc5f116120c4d06bb857407047505a5ca5c526a9c8209cb06411635adc506b9a634024db9152c23f5b98f46d9600d2cf21d4826cf5a947b335cb72449eb2
-  data.tar.gz: e40b8fde694ca637e5ab80d529e161eefe8901cf478a768ebbc2ddd35b9cc16f42e8858feb4cd149333e7e3db6f4ee977770485390c56303ba8dba3fd88803ff
+  metadata.gz: 3b5fce35dda024e1011917c41685c88eadf9537558bcc643c06e4db988fc99dd3819f4d0b5f5be1ff5f660a33053b3c4b6fc579e1723523edc940cef67682d39
+  data.tar.gz: e0025127be66530aa82e64bcc98141b20cf5e5fd0a66290f32bcd88b16b4b7143a144238f85b0f209c7f7fc56477d85cfa3556f5b1808940f31d55086bf7e99d

data/README.md

@@ -64,6 +64,34 @@ SlateDb::Database.open("/tmp/mydb") do |db|
 end  # automatically closed
 ```
 
+### Compaction Controls
+
+SlateDB 0.17 adds controls for how long compaction inputs remain protected from
+garbage collection and when to consolidate sorted runs:
+
+```ruby
+SlateDb::Database.open("mydb",
+                       url: "s3://mybucket/path",
+                       compactor_checkpoint_lifetime: 30 * 60_000,
+                       sorted_run_consolidation_threshold: 8) do |db|
+  db.put("key", "value")
+end
+```
+
+`compactor_checkpoint_lifetime` is in milliseconds and defaults to 15 minutes.
+Reads, scans, snapshots, and transactions using old compaction inputs must finish
+within that time; after it expires, garbage collection can remove those inputs.
+
+`sorted_run_consolidation_threshold` starts fallback consolidation when the
+projected sorted-run count exceeds the threshold. It is not a hard limit.
+Zero disables it (the default); use at least two to enable it. SlateDB warns and
+disables the trigger for one. Low thresholds can increase storage reads and writes.
+
+### Key Size
+
+Keys can now exceed the previous 65,535-byte limit, up to SlateDB's new
+4,294,967,295-byte limit. Very large keys still require enough memory to process.
+
 ### Persistent Storage
 
 For persistent storage, provide an object store URL:
@@ -501,6 +529,14 @@ SlateDb::Reader.open("/tmp/mydb", url: "s3://bucket/path") do |reader|
   reader.scan("prefix:").each do |k, v|
     puts "#{k}: #{v}"
   end
+
+  # Range and prefix scans accept :asc/:ascending or :desc/:descending.
+  reader.scan("user:100", "user:200", order: :desc).each do |k, v|
+    puts "#{k}: #{v}"
+  end
+  reader.scan_prefix("user:", from: "100", to: "200", order: :desc).each do |k, v|
+    puts "#{k}: #{v}"
+  end
 end
 
 # Open at a specific checkpoint (pins the reader to that checkpoint's state)

data/ext/slatedb/Cargo.toml

@@ -11,13 +11,13 @@ name = "slatedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-slatedb = "0.16.0"
-magnus = { version = "0.8.2", features = ["rb-sys"] }
-rb-sys = { version = "0.9.128", features = ["stable-api-compiled-fallback"] }
-tokio = { version = "1.52.3", features = ["rt-multi-thread", "sync"] }
-bytes = "1.11.1"
-serde_json = "1.0.145"
+slatedb = "0.17.0"
+magnus = { version = "0.9.3", features = ["rb-sys"] }
+rb-sys = { version = "0.9.130", features = ["stable-api-compiled-fallback"] }
+tokio = { version = "1.54.1", features = ["rt-multi-thread", "sync"] }
+bytes = "1.12.1"
+serde_json = "1.0.154"
 url = "2.5.8"
 once_cell = "1.21.4"
-log = "0.4.29"
-uuid = "1.23.1"
+log = "0.4.34"
+uuid = "1.29.0"

data/ext/slatedb/src/database.rs

@@ -5,7 +5,7 @@ use magnus::prelude::*;
 use magnus::{function, method, Error, RHash, Ruby};
 use slatedb::config::{
     CloseOptions, DurabilityLevel, FlushOptions, FlushType, MergeOptions, PutOptions, ReadOptions,
-    ScanOptions, Ttl, WriteOptions,
+    ScanOptions, Settings, Ttl, WriteOptions,
 };
 use slatedb::object_store::memory::InMemory;
 use slatedb::{Db, IsolationLevel, IterationOrder, KeyValue};
@@ -120,6 +120,23 @@ impl Database {
     pub fn open(path: String, url: Option<String>, kwargs: RHash) -> Result<Self, Error> {
         // Try string-based merge operator first, then proc-based
         let merge_operator = parse_merge_operator(&kwargs)?.or(parse_merge_operator_proc(&kwargs)?);
+        let checkpoint_lifetime = get_optional::<u64>(&kwargs, "compactor_checkpoint_lifetime")?
+            .map(std::time::Duration::from_millis);
+        let consolidation_threshold =
+            get_optional::<usize>(&kwargs, "sorted_run_consolidation_threshold")?;
+
+        let mut settings = Settings::default();
+        if let Some(compactor) = settings.compactor_options.as_mut() {
+            if let Some(lifetime) = checkpoint_lifetime {
+                compactor.checkpoint_lifetime = lifetime;
+            }
+            if let Some(threshold) = consolidation_threshold {
+                compactor.scheduler_options.insert(
+                    "sorted_run_consolidation_threshold".to_string(),
+                    threshold.to_string(),
+                );
+            }
+        }
 
         let db = block_on_result(async {
             let object_store: Arc<dyn slatedb::object_store::ObjectStore> =
@@ -129,7 +146,7 @@ impl Database {
                     Arc::new(InMemory::new())
                 };
 
-            let mut builder = Db::builder(path, object_store);
+            let mut builder = Db::builder(path, object_store).with_settings(settings);
             if let Some(merge_operator) = merge_operator {
                 builder = builder.with_merge_operator(merge_operator);
             }

data/lib/slatedb/database.rb

@@ -12,6 +12,12 @@ module SlateDb
       # @param merge_operator [Symbol, String, Proc, nil] Optional merge operator.
       #   Can be a symbol/string ("string_concat" or "concat") or a Proc/lambda
       #   that takes (key, existing_value, new_value) and returns the merged value.
+      # @param compactor_checkpoint_lifetime [Integer, nil] How long compaction inputs
+      #   are protected from garbage collection, in milliseconds (default: 15 minutes).
+      #   Reads using those inputs must finish within this time.
+      # @param sorted_run_consolidation_threshold [Integer, nil] Sorted-run count above
+      #   which fallback consolidation starts. Zero disables it (default); use at least
+      #   two to enable it. One is disabled by SlateDB with a warning. This is not a hard limit.
       # @yield [db] If a block is given, yields the database and ensures it's closed
       # @return [Database] The opened database (or block result if block given)
       #
@@ -38,8 +44,12 @@ module SlateDb
       #   db.merge("counter", "3")
       #   db.get("counter") # => "8"
       #
-      def open(path, url: nil, merge_operator: nil)
-        opts = {}
+      def open(path, url: nil, merge_operator: nil,
+               compactor_checkpoint_lifetime: nil, sorted_run_consolidation_threshold: nil)
+        opts = {
+          compactor_checkpoint_lifetime: compactor_checkpoint_lifetime,
+          sorted_run_consolidation_threshold: sorted_run_consolidation_threshold
+        }.compact
 
         case merge_operator
         when Symbol, String

data/lib/slatedb/reader.rb

@@ -106,16 +106,19 @@ module SlateDb
     #
     # @param start_key [String] The start key (inclusive)
     # @param end_key [String, nil] The end key (exclusive)
+    # @param order [Symbol, String, nil] Iteration order (:asc/:ascending or :desc/:descending)
     # @return [Iterator] An iterator over key-value pairs
     #
     def scan(start_key, end_key = nil, durability_filter: nil, dirty: nil,
-             read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, &)
-      opts = {}
-      opts[:durability_filter] = durability_filter.to_s if durability_filter
-      opts[:dirty] = dirty unless dirty.nil?
-      opts[:read_ahead_bytes] = read_ahead_bytes if read_ahead_bytes
-      opts[:cache_blocks] = cache_blocks unless cache_blocks.nil?
-      opts[:max_fetch_tasks] = max_fetch_tasks if max_fetch_tasks
+             read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, order: nil, &)
+      opts = {
+        durability_filter: durability_filter&.to_s,
+        dirty: dirty,
+        read_ahead_bytes: read_ahead_bytes,
+        cache_blocks: cache_blocks,
+        max_fetch_tasks: max_fetch_tasks,
+        order: order&.to_s
+      }.compact
 
       iter = if opts.empty?
                _scan(start_key, end_key)
@@ -138,6 +141,7 @@ module SlateDb
     # @param read_ahead_bytes [Integer, nil] Number of bytes to read ahead
     # @param cache_blocks [Boolean, nil] Whether to cache blocks
     # @param max_fetch_tasks [Integer, nil] Maximum number of fetch tasks
+    # @param order [Symbol, String, nil] Iteration order (:asc/:ascending or :desc/:descending)
     # @param from [String, nil] Inclusive lower bound suffix, appended to the
     #   prefix, to start scanning from (e.g. prefix "user:" with from "100"
     #   starts at "user:100"). Defaults to the start of the prefix.
@@ -146,7 +150,7 @@ module SlateDb
     # @return [Iterator] An iterator over key-value pairs
     #
     def scan_prefix(prefix, durability_filter: nil, dirty: nil,
-                    read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil,
+                    read_ahead_bytes: nil, cache_blocks: nil, max_fetch_tasks: nil, order: nil,
                     from: nil, to: nil, &)
       opts = {
         durability_filter: durability_filter&.to_s,
@@ -154,6 +158,7 @@ module SlateDb
         read_ahead_bytes: read_ahead_bytes,
         cache_blocks: cache_blocks,
         max_fetch_tasks: max_fetch_tasks,
+        order: order&.to_s,
         subrange_from: from,
         subrange_to: to
       }.compact

data/lib/slatedb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SlateDb
-  VERSION = "0.5.0"
+  VERSION = "0.6.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: slatedb
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.1
 platform: ruby
 authors:
 - SlateDB Contributors
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.128
+        version: 0.9.130
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.128
+        version: 0.9.130
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -71,14 +71,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.86'
+        version: '1.92'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.86'
+        version: '1.92'
 description: A cloud-native embedded key-value store built on object storage
 email:
 - slatedb@example.com