lz4rip 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6cc17a63b94ba3943cc64ef1ded04a85994da14d1d305df9adebe10e78bfd798
+  data.tar.gz: 849f7c71658601730ff0f328963148644987798bfe06d7e3878f81eae3b5885b
+SHA512:
+  metadata.gz: bf19aaf5f7f57cbaa50bd1c9674ddae4583d18665495344f19325b095a7298b97fb152625bcab41b6e96a37fa71d8e5bbe1c6c295fb27d2443012312b1d9bd05
+  data.tar.gz: bc6c28546e3ad483b6c97601610fc3625123567b1e70f48059626b0bf710fd6cb2df1b7096def34cf2c04c9142d509411a0fe3b5bc7e48c12a96edbad9e9229e

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+## [Unreleased]
+
+## [0.1.0] - 2026-06-20
+
+- Initial release.
+- `Lz4rip::FrameCodec`: frame-format LZ4 codec (Ractor-shareable).
+- `Lz4rip::BlockCodec`: block-format LZ4 codec with reusable scratch table.
+- `Lz4rip::Dictionary`: immutable value type for LZ4 dictionaries.
+- `Lz4rip::DictTrainer`: COVER-based dictionary trainer.
+- `Lz4rip.compress_bound`: maximum output size for a given input size.

data/Cargo.lock

@@ -0,0 +1,325 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "bindgen"
+version = "0.72.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "clang-sys",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn",
+]
+
+[[package]]
+name = "bitflags"
+version = "2.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
+
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "either"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
+
+[[package]]
+name = "glob"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
+[[package]]
+name = "lz4rip"
+version = "0.1.0"
+dependencies = [
+ "lz4rip 0.8.5",
+ "magnus",
+ "rb-sys",
+]
+
+[[package]]
+name = "lz4rip"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16200eb628c6a5a6e539901f08e5ff4adb9b9522d7201a2dfa36d2060c355b34"
+dependencies = [
+ "lz4rip-core",
+ "lz4rip-decode",
+ "lz4rip-encode",
+ "twox-hash",
+]
+
+[[package]]
+name = "lz4rip-core"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bf2d77a9fc2acbfe6f4b6b0a3ef2d97f0c96bb384c4406225851a900f483788"
+
+[[package]]
+name = "lz4rip-decode"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c45bc51677ce22ebf4d9edabdb658fbd9e4556a93316d423a9e7a6a54108166"
+dependencies = [
+ "lz4rip-core",
+]
+
+[[package]]
+name = "lz4rip-encode"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d0d63726468388972bc99c72dab3bb3fde78096969cd270d84e77c07b5df5a2"
+dependencies = [
+ "lz4rip-core",
+]
+
+[[package]]
+name = "magnus"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b36a5b126bbe97eb0d02d07acfeb327036c6319fd816139a49824a83b7f9012"
+dependencies = [
+ "magnus-macros",
+ "rb-sys",
+ "rb-sys-env",
+ "seq-macro",
+]
+
+[[package]]
+name = "magnus-macros"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rb-sys"
+version = "0.9.128"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45ca28513560e56cfb79a62b1fce363c73af170a182024ce880c77ee9429920a"
+dependencies = [
+ "rb-sys-build",
+]
+
+[[package]]
+name = "rb-sys-build"
+version = "0.9.128"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce04b2c55eff3a21aaa623fcc655d94373238e72cac6b3e1a3641ff31649f99a"
+dependencies = [
+ "bindgen",
+ "lazy_static",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "shell-words",
+ "syn",
+]
+
+[[package]]
+name = "rb-sys-env"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a"
+
+[[package]]
+name = "regex"
+version = "1.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+
+[[package]]
+name = "seq-macro"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
+
+[[package]]
+name = "shell-words"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77"
+
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "syn"
+version = "2.0.118"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "twox-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"

data/Cargo.toml

@@ -0,0 +1,9 @@
+[workspace]
+members  = ["ext/lz4rip"]
+resolver = "2"
+
+[profile.release]
+opt-level     = 3
+lto           = true
+codegen-units = 1
+panic         = "abort"

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Patrik Wenger
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,128 @@
+# lz4rip — Ractor-safe LZ4 for Ruby
+
+[![CI](https://github.com/paddor/lz4rip-rb/actions/workflows/ci.yml/badge.svg)](https://github.com/paddor/lz4rip-rb/actions/workflows/ci.yml)
+[![Gem Version](https://img.shields.io/gem/v/lz4rip?color=e9573f)](https://rubygems.org/gems/lz4rip)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+[![Ruby](https://img.shields.io/badge/Ruby-%3E%3D%204.0-CC342D?logo=ruby&logoColor=white)](https://www.ruby-lang.org)
+
+Ruby bindings for [lz4rip](https://crates.io/crates/lz4rip), a pure-Rust LZ4
+implementation. Built with [magnus](https://github.com/matsadler/magnus) and
+declared Ractor-safe so you can compress from any Ractor without a global lock.
+
+## Features
+
+- **Block codec** with reusable compressor scratch table
+- **Frame codec** for standard `.lz4` frames
+- **Dictionary support** for both block and frame codecs
+- **COVER-based dictionary trainer** (`DictTrainer`)
+- **Ractor-safe**: `FrameCodec` is shareable across Ractors, `BlockCodec` is
+  per-Ractor (mutable scratch state)
+
+## Install
+
+Requires Ruby >= 4.0 and a Rust toolchain (for building the native extension):
+
+```sh
+gem install lz4rip
+```
+
+Or in your Gemfile:
+
+```ruby
+gem "lz4rip"
+```
+
+## Usage
+
+### Frame codec (standard LZ4 frames)
+
+```ruby
+require "lz4rip"
+
+codec = Lz4rip::FrameCodec.new
+compressed = codec.compress("hello world " * 1000)
+original   = codec.decompress(compressed)
+```
+
+### Block codec (raw LZ4 blocks)
+
+```ruby
+codec = Lz4rip::BlockCodec.new
+compressed = codec.compress("hello world " * 1000)
+original   = codec.decompress(compressed, decompressed_size: 12_000)
+```
+
+Block decompression requires the original size up front. This is by design: LZ4
+block format does not store it, so the caller must track it.
+
+### Dictionary compression
+
+```ruby
+dict = Lz4rip::Dictionary.new(bytes: "common log prefix: ")
+codec = Lz4rip::FrameCodec.new(dict: dict)
+
+compressed = codec.compress("common log prefix: event=login user=alice")
+original   = codec.decompress(compressed)
+```
+
+### Dictionary training
+
+```ruby
+trainer = Lz4rip::DictTrainer.new(2048)
+messages.each { |msg| trainer.add_sample(msg) }
+dict_bytes = trainer.train
+
+dict  = Lz4rip::Dictionary.new(bytes: dict_bytes)
+codec = Lz4rip::BlockCodec.new(dict: dict)
+```
+
+### Ractor safety
+
+```ruby
+codec = Lz4rip::FrameCodec.new
+
+ractors = 4.times.map do |i|
+  Ractor.new(codec) do |c|
+    data = "ractor #{Ractor.current} payload " * 100
+    ct   = c.compress(data)
+    raise "mismatch" unless c.decompress(ct) == data
+    :ok
+  end
+end
+
+ractors.each { |r| p r.value }  # => :ok, :ok, :ok, :ok
+```
+
+## API
+
+| Class / Module | Method | Description |
+|---|---|---|
+| `Lz4rip::FrameCodec` | `.new(dict: nil)` | Create a frame codec, optionally with a `Dictionary` or raw `String` dict |
+| | `#compress(string)` | Compress to LZ4 frame |
+| | `#decompress(string)` | Decompress an LZ4 frame |
+| | `#has_dict?` | Whether a dictionary is loaded |
+| | `#id` | Dictionary ID (nil without dict) |
+| | `#size` | Dictionary size in bytes (0 without dict) |
+| `Lz4rip::BlockCodec` | `.new(dict: nil)` | Create a block codec, optionally with a dict `String` |
+| | `#compress(string)` | Compress to raw LZ4 block |
+| | `#decompress(string, decompressed_size:)` | Decompress a raw LZ4 block |
+| | `#has_dict?` | Whether a dictionary is loaded |
+| | `#size` | Internal state size in bytes |
+| `Lz4rip::Dictionary` | `.new(bytes:, id: auto)` | Immutable dictionary value object |
+| | `#bytes` | Frozen binary dict bytes |
+| | `#id` | 32-bit dictionary ID |
+| | `#size` | Dictionary size in bytes |
+| `Lz4rip::DictTrainer` | `.new(max_dict_size)` | Create a trainer (capped at 65535) |
+| | `#add_sample(string)` | Feed a training sample |
+| | `#train` | Consume the trainer, return dict bytes |
+| | `#sample_count` | Number of accepted samples |
+| | `#total_bytes` | Total bytes of accepted samples |
+| | `#trained?` | Whether `#train` has been called |
+| | `#max_dict_size` | Configured max dict size |
+| `Lz4rip` | `.compress_bound(size)` | Max compressed output size for a given input size |
+| | `.block_stream_size` | Internal compressor heap size |
+| `Lz4rip::DecompressError` | | Raised on decompression failure (subclass of `StandardError`) |
+
+## License
+
+[MIT](LICENSE)

data/ext/lz4rip/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name    = "lz4rip"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name       = "lz4rip"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+lz4    = { version = "0.8", package = "lz4rip" }
+magnus = "0.8"
+rb-sys = "0.9"
+
+[build-dependencies]
+rb-sys = "0.9"

data/ext/lz4rip/extconf.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("lz4rip/lz4rip") do |r|
+  r.profile = ENV.fetch("RB_SYS_CARGO_PROFILE", :release).to_sym
+end

data/ext/lz4rip/src/lib.rs

@@ -0,0 +1,392 @@
+use magnus::{
+    exception::ExceptionClass, function, method, prelude::*, r_string::RString, value::Opaque,
+    Error, Ruby,
+};
+use std::cell::RefCell;
+use std::io::{Cursor, Read, Write};
+use std::sync::OnceLock;
+
+use lz4::block::{self, Compressor, Decompressor, DictTrainer};
+use lz4::frame::{BlockMode, FrameDecoder, FrameEncoder, FrameInfo};
+
+const COMPRESSOR_HEAP_SIZE: usize = 8192;
+
+const LZ4_FRAME_MAGIC: [u8; 4] = [0x04, 0x22, 0x4d, 0x18];
+
+static DECOMPRESS_ERROR: OnceLock<Opaque<ExceptionClass>> = OnceLock::new();
+
+fn decompress_error(ruby: &Ruby) -> ExceptionClass {
+    ruby.get_inner(
+        *DECOMPRESS_ERROR
+            .get()
+            .expect("DecompressError not initialized"),
+    )
+}
+
+// ---------- module functions ----------
+
+fn lz4rip_compress_bound(_ruby: &Ruby, size: usize) -> usize {
+    block::get_maximum_output_size(size)
+}
+
+fn lz4rip_block_stream_size(_ruby: &Ruby) -> usize {
+    COMPRESSOR_HEAP_SIZE
+}
+
+// ---------- BlockCodec ----------
+
+#[magnus::wrap(class = "Lz4rip::BlockCodec", free_immediately, size)]
+struct BlockCodec {
+    compressor: Option<RefCell<Compressor>>,
+    decompressor: Option<Decompressor>,
+    dict_len: usize,
+}
+
+fn block_codec_new(_ruby: &Ruby, rb_dict: Option<RString>) -> Result<BlockCodec, Error> {
+    match rb_dict {
+        None => Ok(BlockCodec {
+            compressor: None,
+            decompressor: None,
+            dict_len: 0,
+        }),
+        Some(rb_dict) => {
+            let bytes: Vec<u8> = unsafe { rb_dict.as_slice().to_vec() };
+            Ok(BlockCodec {
+                compressor: Some(RefCell::new(Compressor::with_dict(&bytes))),
+                decompressor: Some(Decompressor::with_dict(&bytes)),
+                dict_len: bytes.len(),
+            })
+        }
+    }
+}
+
+fn block_codec_size(rb_self: &BlockCodec) -> usize {
+    if rb_self.compressor.is_some() {
+        COMPRESSOR_HEAP_SIZE + rb_self.dict_len
+    } else {
+        0
+    }
+}
+
+fn block_codec_has_dict(rb_self: &BlockCodec) -> bool {
+    rb_self.compressor.is_some()
+}
+
+fn block_codec_compress(
+    ruby: &Ruby,
+    rb_self: &BlockCodec,
+    rb_input: RString,
+) -> Result<RString, Error> {
+    let input: &[u8] = unsafe { rb_input.as_slice() };
+
+    let out = match &rb_self.compressor {
+        None => block::compress(input),
+        Some(comp) => comp.borrow_mut().compress(input),
+    };
+
+    Ok(ruby.str_from_slice(&out))
+}
+
+fn block_codec_decompress(
+    ruby: &Ruby,
+    rb_self: &BlockCodec,
+    rb_input: RString,
+    decompressed_size: usize,
+) -> Result<RString, Error> {
+    let compressed: &[u8] = unsafe { rb_input.as_slice() };
+
+    let result = match &rb_self.decompressor {
+        None => block::decompress(compressed, decompressed_size),
+        Some(decomp) => decomp.decompress(compressed, decompressed_size),
+    };
+
+    match result {
+        Ok(data) => Ok(ruby.str_from_slice(&data)),
+        Err(e) => Err(Error::new(
+            decompress_error(ruby),
+            format!("lz4 block decode failed: {e}"),
+        )),
+    }
+}
+
+// ---------- FrameCodec ----------
+
+#[magnus::wrap(class = "Lz4rip::FrameCodec", free_immediately, size)]
+struct FrameCodec {
+    dict: Option<DictBound>,
+}
+
+struct DictBound {
+    bytes: Vec<u8>,
+    id: u32,
+}
+
+fn frame_codec_initialize(
+    _ruby: &Ruby,
+    rb_dict: Option<RString>,
+    id: u32,
+) -> Result<FrameCodec, Error> {
+    let dict = rb_dict.map(|s| {
+        let bytes: Vec<u8> = unsafe { s.as_slice().to_vec() };
+        s.freeze();
+        DictBound { bytes, id }
+    });
+    Ok(FrameCodec { dict })
+}
+
+fn frame_codec_compress(
+    ruby: &Ruby,
+    rb_self: &FrameCodec,
+    rb_input: RString,
+) -> Result<RString, Error> {
+    let input: &[u8] = unsafe { rb_input.as_slice() };
+
+    let buf = Vec::new();
+    let mut enc = match &rb_self.dict {
+        None => {
+            let info = FrameInfo::new().block_mode(BlockMode::Linked);
+            FrameEncoder::with_frame_info(info, buf)
+        }
+        Some(d) => FrameEncoder::with_dictionary(buf, &d.bytes, d.id),
+    };
+
+    enc.write_all(input).map_err(|e| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            format!("lz4 frame compress failed: {e}"),
+        )
+    })?;
+
+    let out = enc.finish().map_err(|e| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            format!("lz4 frame compress failed: {e}"),
+        )
+    })?;
+
+    Ok(ruby.str_from_slice(&out))
+}
+
+fn frame_codec_decompress(
+    ruby: &Ruby,
+    rb_self: &FrameCodec,
+    rb_input: RString,
+) -> Result<RString, Error> {
+    let input: &[u8] = unsafe { rb_input.as_slice() };
+
+    if input.len() < 4 || input[..4] != LZ4_FRAME_MAGIC {
+        return Err(Error::new(
+            decompress_error(ruby),
+            "lz4 frame decode failed: bad magic (input is not an LZ4 frame)",
+        ));
+    }
+
+    let mut dec = match &rb_self.dict {
+        None => FrameDecoder::new(Cursor::new(input)),
+        Some(d) => FrameDecoder::with_dictionary(Cursor::new(input), &d.bytes, d.id),
+    };
+
+    let mut out = Vec::new();
+    dec.read_to_end(&mut out).map_err(|e| {
+        Error::new(
+            decompress_error(ruby),
+            format!("lz4 frame decode failed: {e}"),
+        )
+    })?;
+
+    Ok(ruby.str_from_slice(&out))
+}
+
+fn frame_codec_size(rb_self: &FrameCodec) -> usize {
+    rb_self.dict.as_ref().map_or(0, |d| d.bytes.len())
+}
+
+fn frame_codec_has_dict(rb_self: &FrameCodec) -> bool {
+    rb_self.dict.is_some()
+}
+
+fn frame_codec_id(rb_self: &FrameCodec) -> Option<u32> {
+    rb_self.dict.as_ref().map(|d| d.id)
+}
+
+// ---------- DictTrainer ----------
+
+const LZ4_MAX_DISTANCE: usize = 65535;
+
+#[magnus::wrap(class = "Lz4rip::DictTrainer", free_immediately, size)]
+struct RbDictTrainer {
+    inner: RefCell<Option<DictTrainer>>,
+    max_dict_size: usize,
+}
+
+fn dict_trainer_new(_ruby: &Ruby, max_dict_size: usize) -> RbDictTrainer {
+    let capped = max_dict_size.min(LZ4_MAX_DISTANCE);
+    RbDictTrainer {
+        max_dict_size: capped,
+        inner: RefCell::new(Some(DictTrainer::new(max_dict_size))),
+    }
+}
+
+fn dict_trainer_add_sample(
+    ruby: &Ruby,
+    rb_self: &RbDictTrainer,
+    rb_data: RString,
+) -> Result<(), Error> {
+    let mut borrow = rb_self.inner.borrow_mut();
+    let trainer = borrow.as_mut().ok_or_else(|| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            "DictTrainer already consumed by #train",
+        )
+    })?;
+    let data: &[u8] = unsafe { rb_data.as_slice() };
+    let sample = if data.len() > rb_self.max_dict_size {
+        &data[..rb_self.max_dict_size]
+    } else {
+        data
+    };
+    trainer.add_sample(sample);
+    Ok(())
+}
+
+fn dict_trainer_sample_count(ruby: &Ruby, rb_self: &RbDictTrainer) -> Result<usize, Error> {
+    let borrow = rb_self.inner.borrow();
+    borrow.as_ref().map(|t| t.sample_count()).ok_or_else(|| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            "DictTrainer already consumed by #train",
+        )
+    })
+}
+
+fn dict_trainer_total_bytes(ruby: &Ruby, rb_self: &RbDictTrainer) -> Result<usize, Error> {
+    let borrow = rb_self.inner.borrow();
+    borrow.as_ref().map(|t| t.total_bytes()).ok_or_else(|| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            "DictTrainer already consumed by #train",
+        )
+    })
+}
+
+fn dict_trainer_train(ruby: &Ruby, rb_self: &RbDictTrainer) -> Result<RString, Error> {
+    let trainer = rb_self.inner.borrow_mut().take().ok_or_else(|| {
+        Error::new(
+            ruby.exception_runtime_error(),
+            "DictTrainer already consumed by #train",
+        )
+    })?;
+    let dict = trainer.train();
+    Ok(ruby.str_from_slice(&dict))
+}
+
+fn dict_trainer_max_dict_size(rb_self: &RbDictTrainer) -> usize {
+    rb_self.max_dict_size
+}
+
+fn dict_trainer_trained(rb_self: &RbDictTrainer) -> bool {
+    rb_self.inner.borrow().is_none()
+}
+
+// ---------- module init ----------
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    unsafe { rb_sys::rb_ext_ractor_safe(true) };
+
+    let module = ruby.define_module("Lz4rip")?;
+
+    let decompress_error_class =
+        module.define_error("DecompressError", ruby.exception_standard_error())?;
+    DECOMPRESS_ERROR
+        .set(Opaque::from(decompress_error_class))
+        .unwrap_or_else(|_| panic!("init called more than once"));
+
+    module.define_module_function("compress_bound", function!(lz4rip_compress_bound, 1))?;
+    module.define_module_function("block_stream_size", function!(lz4rip_block_stream_size, 0))?;
+
+    let codec_class = module.define_class("BlockCodec", ruby.class_object())?;
+    codec_class.define_singleton_method("_native_new", function!(block_codec_new, 1))?;
+    codec_class.define_method("size", method!(block_codec_size, 0))?;
+    codec_class.define_method("has_dict?", method!(block_codec_has_dict, 0))?;
+    codec_class.define_method("compress", method!(block_codec_compress, 1))?;
+    codec_class.define_method("_decompress", method!(block_codec_decompress, 2))?;
+
+    let trainer_class = module.define_class("DictTrainer", ruby.class_object())?;
+    trainer_class.define_singleton_method("_native_new", function!(dict_trainer_new, 1))?;
+    trainer_class.define_method("add_sample", method!(dict_trainer_add_sample, 1))?;
+    trainer_class.define_method("sample_count", method!(dict_trainer_sample_count, 0))?;
+    trainer_class.define_method("total_bytes", method!(dict_trainer_total_bytes, 0))?;
+    trainer_class.define_method("train", method!(dict_trainer_train, 0))?;
+    trainer_class.define_method("max_dict_size", method!(dict_trainer_max_dict_size, 0))?;
+    trainer_class.define_method("trained?", method!(dict_trainer_trained, 0))?;
+
+    let frame_codec_class = module.define_class("FrameCodec", ruby.class_object())?;
+    frame_codec_class
+        .define_singleton_method("_native_new", function!(frame_codec_initialize, 2))?;
+    frame_codec_class.define_method("compress", method!(frame_codec_compress, 1))?;
+    frame_codec_class.define_method("decompress", method!(frame_codec_decompress, 1))?;
+    frame_codec_class.define_method("size", method!(frame_codec_size, 0))?;
+    frame_codec_class.define_method("has_dict?", method!(frame_codec_has_dict, 0))?;
+    frame_codec_class.define_method("id", method!(frame_codec_id, 0))?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn block_round_trip() {
+        let data = b"hello hello hello hello".to_vec();
+        let ct = block::compress(&data);
+        let pt = block::decompress(&ct, data.len()).unwrap();
+        assert_eq!(pt, data);
+    }
+
+    #[test]
+    fn block_dict_round_trip() {
+        let dict = b"common log prefix: ".to_vec();
+        let msg = b"common log prefix: event=login user=alice".to_vec();
+
+        let mut comp = Compressor::with_dict(&dict);
+        let ct_dict = comp.compress(&msg);
+        let decomp = Decompressor::with_dict(&dict);
+        let pt = decomp.decompress(&ct_dict, msg.len()).unwrap();
+        assert_eq!(pt, msg);
+
+        let ct_plain = block::compress(&msg);
+        assert!(
+            ct_dict.len() < ct_plain.len(),
+            "dict compression should beat no-dict on shared-prefix input"
+        );
+    }
+
+    #[test]
+    fn frame_round_trip() {
+        let data = b"the quick brown fox jumps over the lazy dog ".repeat(100);
+        let mut enc = FrameEncoder::new(Vec::new());
+        enc.write_all(&data).unwrap();
+        let ct = enc.finish().unwrap();
+        assert!(ct.len() < data.len());
+        assert_eq!(&ct[..4], &LZ4_FRAME_MAGIC);
+
+        let mut dec = FrameDecoder::new(Cursor::new(&ct));
+        let mut pt = Vec::new();
+        dec.read_to_end(&mut pt).unwrap();
+        assert_eq!(pt, data);
+    }
+
+    #[test]
+    fn frame_empty_round_trip() {
+        let mut enc = FrameEncoder::new(Vec::new());
+        enc.write_all(b"").unwrap();
+        let ct = enc.finish().unwrap();
+
+        let mut dec = FrameDecoder::new(Cursor::new(&ct));
+        let mut pt = Vec::new();
+        dec.read_to_end(&mut pt).unwrap();
+        assert!(pt.is_empty());
+    }
+}

data/lib/lz4rip/block_codec.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "dictionary"
+
+module Lz4rip
+  class BlockCodec
+    def self.new(dict: nil)
+      _native_new(Dictionary === dict ? dict.bytes : dict)
+    end
+
+
+    def decompress(bytes, decompressed_size:)
+      _decompress(bytes, decompressed_size)
+    end
+  end
+end

data/lib/lz4rip/dict_trainer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Lz4rip
+  class DictTrainer
+    def self.new(max_dict_size)
+      _native_new(max_dict_size)
+    end
+  end
+end

data/lib/lz4rip/dictionary.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Lz4rip
+  Dictionary = Data.define(:bytes, :id) do
+    def initialize(bytes:, id: Digest::SHA256.digest(bytes).byteslice(0, 4).unpack1("V"))
+      super(bytes: bytes.b.freeze, id: id)
+    end
+
+
+    def size
+      bytes.bytesize
+    end
+  end
+end

data/lib/lz4rip/frame_codec.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative "dictionary"
+
+module Lz4rip
+  class FrameCodec
+    def self.new(dict: nil)
+      case dict
+      when nil
+        _native_new(nil, 0)
+      when Dictionary
+        _native_new(dict.bytes, dict.id)
+      when String
+        _native_new(dict, Dictionary.new(bytes: dict).id)
+      else
+        raise TypeError, "expected Lz4rip::Dictionary, String, or nil; got #{dict.class}"
+      end
+    end
+  end
+end

data/lib/lz4rip/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Lz4rip
+  VERSION = "0.1.0"
+end

data/lib/lz4rip.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "lz4rip/lz4rip"        # Rust extension
+require_relative "lz4rip/version"
+require_relative "lz4rip/dictionary"
+require_relative "lz4rip/block_codec"
+require_relative "lz4rip/frame_codec"
+require_relative "lz4rip/dict_trainer"

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification
+name: lz4rip
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Patrik Wenger
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: Ruby bindings (via Rust/magnus) for lz4rip, a pure-Rust LZ4 implementation.
+  Block-format and frame-format compress/decompress with optional dictionary support
+  and COVER-based dictionary training. Ractor-safe.
+email:
+- paddor@gmail.com
+executables: []
+extensions:
+- ext/lz4rip/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Cargo.lock
+- Cargo.toml
+- LICENSE
+- README.md
+- ext/lz4rip/Cargo.toml
+- ext/lz4rip/extconf.rb
+- ext/lz4rip/src/lib.rs
+- lib/lz4rip.rb
+- lib/lz4rip/block_codec.rb
+- lib/lz4rip/dict_trainer.rb
+- lib/lz4rip/dictionary.rb
+- lib/lz4rip/frame_codec.rb
+- lib/lz4rip/version.rb
+homepage: https://github.com/paddor/lz4rip-rb
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/paddor/lz4rip-rb
+  source_code_uri: https://github.com/paddor/lz4rip-rb
+  changelog_uri: https://github.com/paddor/lz4rip-rb/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 4.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.10
+specification_version: 4
+summary: Ractor-safe LZ4 compression for Ruby
+test_files: []