rlz4 0.1.1 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc8e8cccbcbf8fd18231558a8e9486e630dadd9287f6a424d3565790ec2b28a4
-  data.tar.gz: e99944e60c12edb16ced111597ee628274c5fee7f977d6d6f1a57ca2217bfefa
+  metadata.gz: 8393efee154a550c5eb3889849ccb3d39867cac460c19176ea7a11d1e4be1595
+  data.tar.gz: 59cbb3f50ab09db3c9e252634da446448436581dd72767d6d9e5c4f9d164f27a
 SHA512:
-  metadata.gz: f548b432680ad0e5c12ae606696049f3710be1d26211973103d0607ee9241af8d786a95c8eb12e4bf5b52230387a9873bfd524913236f7be3a04672c98130b2b
-  data.tar.gz: f745e2487bff140a571d18670b23b22b77acfcaa66174bfa72cb740efdc2c708905414c1fce3ce6e31fe634446c523c2e257cbcecddc74babdd956634f916a68
+  metadata.gz: 9eb27c1712c68e370697eb3edb1e92ff311769651605f04b0157ed180d0419467e9d4e0432fc4d07b80dca4d8cbcc3e349888bd81fd73321e806dac87d870736
+  data.tar.gz: 30ac184b6921cee53e17c064bddaadea96e39789b38893d68a8cfd8816531d1964fdc30ed53a28a1641a3e32670deab2c8f403d4a85584efb8fc4f66bc5a1eab

data/Cargo.lock

@@ -107,8 +107,7 @@ dependencies = [
 [[package]]
 name = "lz4_flex"
 version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a"
+source = "git+https://github.com/paddor/lz4_flex.git?rev=dae9c784e890591e6445135ba23cacf344eafe8f#dae9c784e890591e6445135ba23cacf344eafe8f"
 dependencies = [
  "twox-hash",
 ]
@@ -237,7 +236,7 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
 
 [[package]]
 name = "rlz4"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "lz4_flex",
  "magnus",

data/README.md

@@ -55,9 +55,10 @@ end
 
 For workloads where many small messages share a common prefix (e.g. ZMQ
 messages with a fixed header), a shared dictionary massively improves the
-compression ratio. `RLZ4::Dictionary` uses LZ4 **block** format with the
-original size prepended — this is a different wire format from
-`RLZ4.compress` and is not interoperable with it.
+compression ratio. `RLZ4::Dictionary#compress` emits a **real LZ4 frame**
+(magic `04 22 4D 18`) with the `FLG.DictID` bit set and the dictionary's
+`Dict_ID` written into the FrameDescriptor — interoperable with the
+reference `lz4` CLI given the same dictionary file (`lz4 -d -D dict.bin`).
 
 ```ruby
 dict = RLZ4::Dictionary.new("schema=v1 type=message field1=")
@@ -66,11 +67,39 @@ compressed   = dict.compress("schema=v1 type=message field1=payload")
 decompressed = dict.decompress(compressed)
 
 dict.size  # => 30
+dict.id    # => u32 Dict_ID
 ```
 
 `RLZ4::Dictionary` is immutable after construction and can be shared across
 Ractors.
 
+## Dictionary IDs
+
+`Dictionary#id` is a `u32` derived from `sha256(dict_bytes)[0..4]`
+interpreted little-endian. The LZ4 frame spec defines `Dict_ID` as
+an application-defined field with no reserved ranges and no central
+registrar, so the full `u32` space is usable.
+
+The id **is on the wire**: `Dictionary#compress` sets `FLG.DictID = 1`
+and writes the id into the FrameDescriptor. On decode, `rlz4` parses
+the incoming frame's `Dict_ID` and asserts it matches
+`Dictionary#id` before touching the payload. Receivers that maintain
+multiple dictionaries can therefore route incoming frames to the
+right one purely by parsing the frame header — no out-of-band id
+channel needed.
+
+LZ4 dictionaries are always raw bytes (unlike Zstd, there is no
+dict-file header format), so there is no header to parse an id out
+of. If you need sender and receiver to agree on an id without
+shipping it out-of-band, deriving it deterministically from the
+dict bytes — which is what `Dictionary.new` does — is the simplest
+option.
+
+Dictionary training from a sample corpus is **not supported**: LZ4
+has no equivalent of Zstd's `ZDICT_trainFromBuffer`. Dictionaries
+are supplied by the caller as raw bytes (typically a hand-picked
+prefix or a representative message).
+
 ### Ractors
 
 Both the module functions and `RLZ4::Dictionary` can be used from any

data/ext/rlz4/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name    = "rlz4"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 
 [lib]
@@ -8,7 +8,7 @@ name       = "rlz4"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-lz4_flex = { version = "0.13", default-features = false, features = ["frame", "std", "safe-encode", "safe-decode"] }
+lz4_flex = { git = "https://github.com/paddor/lz4_flex.git", rev = "dae9c784e890591e6445135ba23cacf344eafe8f", default-features = false, features = ["frame", "std", "safe-encode", "safe-decode"] }
 magnus   = "0.8"
 rb-sys   = "0.9"
 

data/ext/rlz4/src/lib.rs

@@ -78,35 +78,56 @@ fn rlz4_decompress(ruby: &Ruby, rb_input: RString) -> Result<RString, Error> {
     Ok(ruby.str_from_slice(&out))
 }
 
-// ---------- Dictionary: block-format compression with a shared dictionary ----------
+// ---------- Dictionary: dict-bound LZ4 frame compression ----------
 //
-// lz4_flex's frame format does not implement dictionary-based compression
-// (FrameInfo::dict_id is metadata-only). For the small-ZMQ-message use case
-// that motivates this class, block format with a prepended size is a better
-// fit anyway: lower per-message overhead and direct dictionary support.
+// Backed by lz4_flex's `FrameEncoder::with_dictionary` /
+// `FrameDecoder::with_dictionary` (added in our fork). Output is a real
+// LZ4 frame with the FLG.DictID bit set and `Dict_ID` written into the
+// FrameDescriptor — interoperable with the reference `lz4` CLI given the
+// same dictionary file.
 //
-// Output is a raw LZ4 block with the original (uncompressed) size prepended
-// as a little-endian u32, matching lz4_flex's `*_size_prepended` API.
+// `Dict_ID` is supplied by the caller (the Ruby wrapper in `lib/rlz4.rb`
+// derives it from `sha256(dict_bytes)[0..4]` interpreted little-endian).
+// Doing the digest in Ruby keeps a hash crate out of the Rust extension's
+// dependency tree.
 #[magnus::wrap(class = "RLZ4::Dictionary", free_immediately, size)]
 struct Dictionary {
     bytes: Vec<u8>,
+    id: u32,
 }
 
-// Safety: Dictionary is read-only after construction (just a byte buffer).
-// No interior mutability, no references to thread-local data.
+// Safety: Dictionary is read-only after construction (just a byte buffer
+// plus a derived id). No interior mutability, no thread-local refs.
 unsafe impl Send for Dictionary {}
 unsafe impl Sync for Dictionary {}
 
-fn dict_initialize(_ruby: &Ruby, rb_dict: RString) -> Result<Dictionary, Error> {
+fn dict_initialize(_ruby: &Ruby, rb_dict: RString, id: u32) -> Result<Dictionary, Error> {
     // SAFETY: copy bytes into an owned Vec before any Ruby allocation.
     let bytes: Vec<u8> = unsafe { rb_dict.as_slice().to_vec() };
     rb_dict.freeze();
-    Ok(Dictionary { bytes })
+    Ok(Dictionary { bytes, id })
 }
 
 fn dict_compress(ruby: &Ruby, rb_self: &Dictionary, rb_input: RString) -> Result<RString, Error> {
     let input: Vec<u8> = unsafe { rb_input.as_slice().to_vec() };
-    let compressed = lz4_flex::block::compress_prepend_size_with_dict(&input, &rb_self.bytes);
+    let upper = lz4_flex::block::get_maximum_output_size(input.len()) + 64;
+    let mut encoder = lz4_flex::frame::FrameEncoder::with_dictionary(
+        Vec::with_capacity(upper),
+        &rb_self.bytes,
+        rb_self.id,
+    );
+    encoder.write_all(&input).map_err(|e| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            format!("lz4 dict frame encode write failed: {e}"),
+        )
+    })?;
+    let compressed = encoder.finish().map_err(|e| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            format!("lz4 dict frame encode finish failed: {e}"),
+        )
+    })?;
     Ok(ruby.str_from_slice(&compressed))
 }
 
@@ -116,13 +137,25 @@ fn dict_decompress(
     rb_input: RString,
 ) -> Result<RString, Error> {
     let compressed: Vec<u8> = unsafe { rb_input.as_slice().to_vec() };
-    let out = lz4_flex::block::decompress_size_prepended_with_dict(&compressed, &rb_self.bytes)
-        .map_err(|e| {
-            Error::new(
-                decompress_error(ruby),
-                format!("lz4 block decode failed: {e}"),
-            )
-        })?;
+    if compressed.len() < LZ4_FRAME_MAGIC.len() || compressed[..4] != LZ4_FRAME_MAGIC {
+        return Err(Error::new(
+            decompress_error(ruby),
+            "lz4 dict frame decode failed: bad magic (input is not an LZ4 frame)",
+        ));
+    }
+
+    let mut decoder = lz4_flex::frame::FrameDecoder::with_dictionary(
+        &compressed[..],
+        &rb_self.bytes,
+        rb_self.id,
+    );
+    let mut out = Vec::new();
+    decoder.read_to_end(&mut out).map_err(|e| {
+        Error::new(
+            decompress_error(ruby),
+            format!("lz4 dict frame decode failed: {e}"),
+        )
+    })?;
     Ok(ruby.str_from_slice(&out))
 }
 
@@ -130,6 +163,10 @@ fn dict_size(rb_self: &Dictionary) -> usize {
     rb_self.bytes.len()
 }
 
+fn dict_id(rb_self: &Dictionary) -> u32 {
+    rb_self.id
+}
+
 // ---------- module init ----------
 
 #[magnus::init]
@@ -152,10 +189,13 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     module.define_module_function("decompress", function!(rlz4_decompress, 1))?;
 
     let dict_class = module.define_class("Dictionary", ruby.class_object())?;
-    dict_class.define_singleton_method("new", function!(dict_initialize, 1))?;
+    // Bound as `_native_new(bytes, id)`. Ruby's `RLZ4::Dictionary.new(bytes)`
+    // computes the id and forwards — see `lib/rlz4.rb`.
+    dict_class.define_singleton_method("_native_new", function!(dict_initialize, 2))?;
     dict_class.define_method("compress", method!(dict_compress, 1))?;
     dict_class.define_method("decompress", method!(dict_decompress, 1))?;
     dict_class.define_method("size", method!(dict_size, 0))?;
+    dict_class.define_method("id", method!(dict_id, 0))?;
 
     Ok(())
 }
@@ -202,25 +242,36 @@ mod tests {
     }
 
     #[test]
-    fn block_dict_round_trip() {
-        let dict = b"JSON schema version 1 field ";
-        let msg = b"JSON schema version 1 field name=hello value=world";
-        let ct = lz4_flex::block::compress_prepend_size_with_dict(msg, dict);
-        let pt = lz4_flex::block::decompress_size_prepended_with_dict(&ct, dict).unwrap();
+    fn frame_dict_round_trip() {
+        let dict = b"JSON schema version 1 field ".repeat(4);
+        let id: u32 = 0xDEAD_BEEF;
+        let msg = b"JSON schema version 1 field name=hello value=world".to_vec();
+
+        let mut enc = lz4_flex::frame::FrameEncoder::with_dictionary(Vec::new(), &dict, id);
+        enc.write_all(&msg).unwrap();
+        let ct = enc.finish().unwrap();
+        assert_eq!(&ct[..4], &[0x04, 0x22, 0x4d, 0x18]);
+
+        let mut dec = lz4_flex::frame::FrameDecoder::with_dictionary(&*ct, &dict, id);
+        let mut pt = Vec::new();
+        dec.read_to_end(&mut pt).unwrap();
         assert_eq!(pt, msg);
     }
 
     #[test]
-    fn block_dict_mismatch_fails_or_returns_wrong_data() {
-        // With a wrong dict, decode either errors out or returns wrong bytes.
-        // Either way it must not silently round-trip to the original.
-        let dict_a = b"common prefix AAA ";
-        let dict_b = b"common prefix BBB ";
+    fn frame_dict_id_mismatch_fails() {
+        let dict_a = b"common prefix AAA ".repeat(4);
+        let dict_b = b"common prefix BBB ".repeat(4);
+
         let msg = b"common prefix AAA : the payload";
-        let ct = lz4_flex::block::compress_prepend_size_with_dict(msg, dict_a);
-        match lz4_flex::block::decompress_size_prepended_with_dict(&ct, dict_b) {
-            Ok(out) => assert_ne!(out, msg),
-            Err(_) => {}
-        }
+        let mut enc =
+            lz4_flex::frame::FrameEncoder::with_dictionary(Vec::new(), &dict_a, 0xAAAA_AAAA);
+        enc.write_all(msg).unwrap();
+        let ct = enc.finish().unwrap();
+
+        let mut dec =
+            lz4_flex::frame::FrameDecoder::with_dictionary(&*ct, &dict_b, 0xBBBB_BBBB);
+        let mut out = Vec::new();
+        assert!(dec.read_to_end(&mut out).is_err());
     }
 }

data/lib/rlz4/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RLZ4
-  VERSION = "0.1.1"
+  VERSION = "0.2.1"
 end

data/lib/rlz4.rb

@@ -1,4 +1,20 @@
 # frozen_string_literal: true
 
+require "digest"
+
 require_relative "rlz4/rlz4"
 require_relative "rlz4/version"
+
+module RLZ4
+  class Dictionary
+    # Public constructor. Derives the LZ4 frame `Dict_ID` from the dictionary
+    # bytes (sha256 truncated to the first 4 bytes, little-endian) and forwards
+    # to the Rust extension. The id is what gets written into every emitted
+    # frame's FrameDescriptor and what `#decompress` asserts the incoming
+    # frame declares before decoding.
+    def self.new(bytes)
+      id = Digest::SHA256.digest(bytes).byteslice(0, 4).unpack1("V")
+      _native_new(bytes, id)
+    end
+  end
+end

data/tmp/x86_64-linux/stage/ext/rlz4/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name    = "rlz4"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 
 [lib]
@@ -8,7 +8,7 @@ name       = "rlz4"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-lz4_flex = { version = "0.13", default-features = false, features = ["frame", "std", "safe-encode", "safe-decode"] }
+lz4_flex = { git = "https://github.com/paddor/lz4_flex.git", rev = "dae9c784e890591e6445135ba23cacf344eafe8f", default-features = false, features = ["frame", "std", "safe-encode", "safe-decode"] }
 magnus   = "0.8"
 rb-sys   = "0.9"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rlz4
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
 platform: ruby
 authors:
 - Patrik Wenger