rlz4 0.2.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8393efee154a550c5eb3889849ccb3d39867cac460c19176ea7a11d1e4be1595
-  data.tar.gz: 59cbb3f50ab09db3c9e252634da446448436581dd72767d6d9e5c4f9d164f27a
+  metadata.gz: 2c9d2822432c3875768ab9ca26a5c95b148f5be3d08e1687e49c131dd3d304a3
+  data.tar.gz: 15726fc411ec89b05dda6c4b41af0ee9a99553d30c45ed0b9a7dab9a76b758da
 SHA512:
-  metadata.gz: 9eb27c1712c68e370697eb3edb1e92ff311769651605f04b0157ed180d0419467e9d4e0432fc4d07b80dca4d8cbcc3e349888bd81fd73321e806dac87d870736
-  data.tar.gz: 30ac184b6921cee53e17c064bddaadea96e39789b38893d68a8cfd8816531d1964fdc30ed53a28a1641a3e32670deab2c8f403d4a85584efb8fc4f66bc5a1eab
+  metadata.gz: 91ab2e6a86d5e7b68be6055848752e871c0baf3d0a1eabf10cbb2624960f6ad6a73d669b3264a3dc1631082efdf02e1f441c303331447b23f0f412a091943142
+  data.tar.gz: 727ee0cd74c5974094f7ccb25257254cd1ee38605aa89aab513fc133bd429e09e9b7c6c09e59f91b6a4161bded800d4d18324973f397a3e50567805af999cac2

data/Cargo.lock

@@ -35,6 +35,16 @@ version = "2.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
 
+[[package]]
+name = "cc"
+version = "1.2.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
 [[package]]
 name = "cexpr"
 version = "0.6.0"
@@ -67,6 +77,12 @@ version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+
 [[package]]
 name = "glob"
 version = "0.3.3"
@@ -105,11 +121,13 @@ dependencies = [
 ]
 
 [[package]]
-name = "lz4_flex"
-version = "0.13.0"
-source = "git+https://github.com/paddor/lz4_flex.git?rev=dae9c784e890591e6445135ba23cacf344eafe8f#dae9c784e890591e6445135ba23cacf344eafe8f"
+name = "lz4-sys"
+version = "1.11.1+lz4-1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6"
 dependencies = [
- "twox-hash",
+ "cc",
+ "libc",
 ]
 
 [[package]]
@@ -236,9 +254,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
 
 [[package]]
 name = "rlz4"
-version = "0.2.0"
+version = "0.4.0"
 dependencies = [
- "lz4_flex",
+ "lz4-sys",
  "magnus",
  "rb-sys",
 ]
@@ -278,12 +296,6 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "twox-hash"
-version = "2.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"

data/README.md

@@ -27,91 +27,110 @@ gem "rlz4"
 
 Building requires a Rust toolchain (stable).
 
-## Usage
+## API
 
-### Frame format (module functions)
+Three classes plus one utility module function:
 
-```ruby
-require "rlz4"
+| | Purpose | Wire format |
+|---|---|---|
+| `RLZ4::Dictionary` | Value type: dict bytes + 4-byte id | — |
+| `RLZ4::FrameCodec` | Optionally dict-bound frame codec | LZ4 frame (`04 22 4D 18`), interoperable with `lz4` CLI |
+| `RLZ4::BlockCodec` | Optionally dict-bound block codec, reusable scratch | Raw LZ4 block, no framing |
+| `RLZ4.compress_bound(n)` | Worst-case output size for input size `n` | — |
 
-compressed   = RLZ4.compress("hello world" * 100)
-decompressed = RLZ4.decompress(compressed)
+Invalid input on decompress raises `RLZ4::DecompressError`
+(a `StandardError` subclass).
 
-# Wire format is standard LZ4 frame (magic number 04 22 4D 18),
-# interoperable with any other LZ4 frame implementation.
-```
+## RLZ4::Dictionary
 
-Invalid input raises `RLZ4::DecompressError` (a `StandardError` subclass):
+Pure value type — just the dict bytes plus a 4-byte id. Built on
+`Data.define`, so it's immutable, has value equality, and is
+shareable across `Ractor`s. The id defaults to `sha256(bytes)[0, 4]`
+interpreted little-endian (the derivation LZ4 frame `FLG.DictID`
+uses); override with `id:` if you need a coordinated value.
 
 ```ruby
-begin
-  RLZ4.decompress("not a valid lz4 frame")
-rescue RLZ4::DecompressError => e
-  warn e.message
-end
+dict = RLZ4::Dictionary.new(bytes: "schema=v1 type=message field1=")
+dict.bytes  # => "schema=v1..." frozen binary
+dict.id     # => u32
+dict.size   # => 30
+
+# With a caller-supplied id (e.g. from an out-of-band protocol):
+custom = RLZ4::Dictionary.new(bytes: raw, id: 0xDEAD_BEEF)
 ```
 
-### Dictionary compression
+## RLZ4::FrameCodec — frame-format LZ4
+
+Emits a real LZ4 frame (magic `04 22 4D 18`), interoperable with the
+`lz4` CLI. With a dictionary, sets `FLG.DictID` and writes `Dict_ID`
+into the FrameDescriptor — a receiver routing by id can pick the
+right dict from a set purely by parsing the frame header.
 
-For workloads where many small messages share a common prefix (e.g. ZMQ
-messages with a fixed header), a shared dictionary massively improves the
-compression ratio. `RLZ4::Dictionary#compress` emits a **real LZ4 frame**
-(magic `04 22 4D 18`) with the `FLG.DictID` bit set and the dictionary's
-`Dict_ID` written into the FrameDescriptor — interoperable with the
-reference `lz4` CLI given the same dictionary file (`lz4 -d -D dict.bin`).
+Stateless (no scratch), so `FrameCodec` instances are shareable
+across `Ractor`s.
 
 ```ruby
-dict = RLZ4::Dictionary.new("schema=v1 type=message field1=")
+codec = RLZ4::FrameCodec.new                           # no dict
+codec = RLZ4::FrameCodec.new(dict: dict)               # Dictionary value
+codec = RLZ4::FrameCodec.new(dict: "raw bytes here")   # String shortcut
 
-compressed   = dict.compress("schema=v1 type=message field1=payload")
-decompressed = dict.decompress(compressed)
+ct = codec.compress("hello world" * 100)
+pt = codec.decompress(ct)
 
-dict.size  # => 30
-dict.id    # => u32 Dict_ID
+codec.has_dict?  # => true / false
+codec.id         # => u32 id when dict-bound, nil otherwise
+codec.size       # => dict size when dict-bound, 0 otherwise
 ```
 
-`RLZ4::Dictionary` is immutable after construction and can be shared across
-Ractors.
+Dict id mismatch on decompress raises `RLZ4::DecompressError`
+before touching the payload — no silently corrupt output.
 
-## Dictionary IDs
+## RLZ4::BlockCodec — block-format LZ4
 
-`Dictionary#id` is a `u32` derived from `sha256(dict_bytes)[0..4]`
-interpreted little-endian. The LZ4 frame spec defines `Dict_ID` as
-an application-defined field with no reserved ranges and no central
-registrar, so the full `u32` space is usable.
+For hot paths that compress many small messages and want to amortise
+allocation. Emits a raw LZ4 block — no frame header, no end-mark,
+no checksum. Not interoperable with the reference `lz4` CLI; meant
+for callers who carry their own framing (e.g. ZMTP transports).
 
-The id **is on the wire**: `Dictionary#compress` sets `FLG.DictID = 1`
-and writes the id into the FrameDescriptor. On decode, `rlz4` parses
-the incoming frame's `Dict_ID` and asserts it matches
-`Dictionary#id` before touching the payload. Receivers that maintain
-multiple dictionaries can therefore route incoming frames to the
-right one purely by parsing the frame header — no out-of-band id
-channel needed.
+Wraps a reusable 16 KiB scratch hash table. With a dictionary, also
+carries a pristine dict-loaded table and restores it into the scratch
+via a single 16 KiB `memcpy` before each compress call — so dict
+initialisation is paid once at construction, not per call.
 
-LZ4 dictionaries are always raw bytes (unlike Zstd, there is no
-dict-file header format), so there is no header to parse an id out
-of. If you need sender and receiver to agree on an id without
-shipping it out-of-band, deriving it deterministically from the
-dict bytes — which is what `Dictionary.new` does — is the simplest
-option.
+```ruby
+codec = RLZ4::BlockCodec.new                           # no dict
+codec = RLZ4::BlockCodec.new(dict: dict)               # Dictionary value
+codec = RLZ4::BlockCodec.new(dict: "raw bytes here")   # String shortcut
 
-Dictionary training from a sample corpus is **not supported**: LZ4
-has no equivalent of Zstd's `ZDICT_trainFromBuffer`. Dictionaries
-are supplied by the caller as raw bytes (typically a hand-picked
-prefix or a representative message).
+ct = codec.compress("hello world" * 100)
+pt = codec.decompress(ct, decompressed_size: 1100)
+```
+
+`#decompress` requires `decompressed_size:` because raw LZ4 blocks
+carry no length prefix. The decoder refuses to write past that
+value even on crafted malformed input — raises
+`RLZ4::DecompressError` on any overrun.
+
+Use `RLZ4.compress_bound(n)` to pre-size output buffers.
 
-### Ractors
+`BlockCodec` holds a `RefCell` internally and is **thread-local** —
+do not cross `Ractor` boundaries. Allocate one per `Ractor`. The
+block format has no on-wire `Dict_ID` field; a dict mismatch
+produces garbage plaintext (not an error). Detect at a higher
+layer (checksum, schema validation, etc.).
 
-Both the module functions and `RLZ4::Dictionary` can be used from any
-Ractor. Example from the test suite:
+## Ractor safety
+
+`Dictionary` and `FrameCodec` can be used from any `Ractor`. Example:
 
 ```ruby
 ractors = 4.times.map do |i|
   Ractor.new(i) do |idx|
-    pt = "ractor #{idx} payload " * 1000
+    codec = RLZ4::FrameCodec.new
+    pt    = "ractor #{idx} payload " * 1000
     1000.times do
-      ct = RLZ4.compress(pt)
-      raise "mismatch" unless RLZ4.decompress(ct) == pt
+      ct = codec.compress(pt)
+      raise "mismatch" unless codec.decompress(ct) == pt
     end
     :ok
   end
@@ -119,11 +138,17 @@ end
 ractors.map(&:value) # => [:ok, :ok, :ok, :ok]
 ```
 
+`BlockCodec` must not cross `Ractor` boundaries — allocate one per
+`Ractor`.
+
 ## Non-goals
 
 - High-compression mode (LZ4_HC).
 - Streaming / chunked compression.
 - Preservation of string encoding on decompress (output is always binary).
+- Dictionary training from a sample corpus. LZ4 has no equivalent of
+  Zstd's `ZDICT_trainFromBuffer`. Dictionaries are caller-supplied
+  raw bytes.
 
 ## License
 

data/ext/rlz4/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name    = "rlz4"
-version = "0.2.0"
+version = "0.4.0"
 edition = "2021"
 
 [lib]
@@ -8,7 +8,7 @@ name       = "rlz4"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-lz4_flex = { git = "https://github.com/paddor/lz4_flex.git", rev = "dae9c784e890591e6445135ba23cacf344eafe8f", default-features = false, features = ["frame", "std", "safe-encode", "safe-decode"] }
+lz4-sys  = "1.11"
 magnus   = "0.8"
 rb-sys   = "0.9"