lz4_flex 0.1.1.pre2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cb58450d76e8b265b117c1760b74dce5286451dc5fc99c20d66c33c763fbace1
+  data.tar.gz: 993a74df952364f4773f4afeeea9763e5f690c3b98a26e7914e5e2a20c9d4de4
+SHA512:
+  metadata.gz: 2f750064cd574bf2d7220aecc37b21d5413aa0d1dbeef15038be2831798966287215613717db0e645b17f48fa80be8baa3b08021870b70e8326ff22cf95a574c
+  data.tar.gz: 718b5b9b0cf4f47a1b333c78e4c38e9ea680ca22333e146408e7aa5aeefab4f340a3915cd9302e2fb34c59e3ca137234160b780fc7ea4f8b4869670533504ae6

data/Cargo.lock

@@ -0,0 +1,510 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "atomic-polyfill"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4"
+dependencies = [
+ "critical-section",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+
+[[package]]
+name = "bindgen"
+version = "0.69.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "clang-sys",
+ "itertools",
+ "lazy_static",
+ "lazycell",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn",
+]
+
+[[package]]
+name = "bitflags"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "cobs"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
+
+[[package]]
+name = "critical-section"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216"
+
+[[package]]
+name = "either"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
+
+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
+[[package]]
+name = "hash32"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67"
+dependencies = [
+ "byteorder",
+]
+
+[[package]]
+name = "heapless"
+version = "0.7.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f"
+dependencies = [
+ "atomic-polyfill",
+ "hash32",
+ "rustc_version",
+ "serde",
+ "spin",
+ "stable_deref_trait",
+]
+
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
+[[package]]
+name = "libc"
+version = "0.2.155"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+
+[[package]]
+name = "libloading"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
+dependencies = [
+ "cfg-if",
+ "windows-targets",
+]
+
+[[package]]
+name = "lock_api"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "lz4_flex"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
+dependencies = [
+ "twox-hash",
+]
+
+[[package]]
+name = "lz4_flex_ext"
+version = "0.1.0"
+dependencies = [
+ "lz4_flex",
+ "magnus",
+ "postcard",
+ "rb-sys",
+ "serde",
+ "static_assertions",
+]
+
+[[package]]
+name = "magnus"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d87ae53030f3a22e83879e666cb94e58a7bdf31706878a0ba48752994146dab"
+dependencies = [
+ "magnus-macros",
+ "rb-sys",
+ "rb-sys-env",
+ "seq-macro",
+]
+
+[[package]]
+name = "magnus-macros"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "postcard"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8"
+dependencies = [
+ "cobs",
+ "heapless",
+ "serde",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.86"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rb-sys"
+version = "0.9.101"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ba2704ccfa7875c91792c57a9aa7c3caac524d3036c122e36eeddad6f6e7c6f"
+dependencies = [
+ "rb-sys-build",
+]
+
+[[package]]
+name = "rb-sys-build"
+version = "0.9.101"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c73585ec80c217b7a81257ca9bb89b191b5e452ec4b9106dc4c2e4e96a822242"
+dependencies = [
+ "bindgen",
+ "lazy_static",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "shell-words",
+ "syn",
+]
+
+[[package]]
+name = "rb-sys-env"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
+
+[[package]]
+name = "regex"
+version = "1.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
+
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
+[[package]]
+name = "rustc_version"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+dependencies = [
+ "semver",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "semver"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
+
+[[package]]
+name = "seq-macro"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
+
+[[package]]
+name = "serde"
+version = "1.0.208"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.208"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "shell-words"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
+
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+dependencies = [
+ "lock_api",
+]
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
+[[package]]
+name = "syn"
+version = "2.0.74"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "twox-hash"
+version = "1.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
+dependencies = [
+ "cfg-if",
+ "static_assertions",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

data/Cargo.toml

@@ -0,0 +1,12 @@
+# This Cargo.toml is here to let externals tools (IDEs, etc.) know that this is
+# a Rust project. Your extensions dependencies should be added to the Cargo.toml
+# in the ext/ directory.
+
+[workspace]
+members = ["./ext/lz4_flex_ext"]
+resolver = "2"
+
+[profile.release]
+debug = true
+codegen-units = 1
+lto = "fat"

data/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Shopify
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# `lz4-flex-rb`
+
+## About this library
+
+**Introduction:**
+`lz4-flex-rb` is a modern LZ4 compression library for Ruby, leveraging the
+power of the [`lz4_flex`](https://github.com/PSeitz/lz4_flex) Rust crate. This
+library provides a pure Rust implementation of the LZ4 algorithm, ensuring high
+performance and safety. One of the standout features of `lz4_flex-rb` is its
+ability to conditionally unlock the Global VM Lock (GVL) for threaded web
+servers, enhancing concurrency and performance in multi-threaded environments.
+
+## How to install this library
+
+### Requirements
+- Ruby 3.0 or higher
+- Rust (for building the native extension)
+
+### Setup
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'lz4_flex'
+```
+
+And then execute:
+
+```sh
+bundle install
+```
+
+### Troubleshooting
+
+If you encounter issues during installation, ensure that Rust is correctly installed and available in your PATH. You can install Rust from [rustup.rs](https://rustup.rs/).
+
+## How to use this library
+
+There are two methods provided, `LZ4Flex.compress` and `Lz4Flex.decompress`.
+Both of these methods utilize the lz4 block format, with a small header to
+record the string's size and encoding.
+
+### Basic Usage
+
+```ruby
+require 'lz4_flex'
+
+# Compress data
+compressed = LZ4Flex.compress("Hello, World!")
+
+# Decompress data
+decompressed = LZ4Flex.decompress(compressed)
+
+puts decompressed  # => "Hello, World!"
+puts decompressed.encoding # => Encoding::UTF_8
+```
+
+The header used in these methods will not be recognizable from other lz4 block
+parsers. If you need that, it's best to use the Frame API (which is currently a
+WIP).
+
+### Migrating from `lz4-ruby`
+
+The [`lz4-ruby`](https://github.com/komiya-atsushi/lz4-ruby) gem uses a slighty
+different header format, which keeps track of string size but not the string's
+encoding.
+
+To make it easy to migrate to `lz4_flex`, we provide a parser for the
+`lz4-ruby` format:
+
+```ruby
+# Say you have a string that was compressed with lz4-ruby...
+lz4_ruby_compressed = LZ4.compress("Yo!")
+
+# You can decode it with lz4_flex:
+decompressed = Lz4Flex::VarInt.decompress(lz4_ruby_compressed)
+
+puts decompressed #=> "Yo!"
+puts decompressed.encoding #=> Encoding::BINARY
+```
+
+Combine
+
+### Running Tests
+To run the tests, execute:
+
+```sh
+bundle exec rake
+```
+
+## Contribute to this library (optional)
+
+1. Fork the repository.
+2. Create a new branch (`git checkout -b my-feature-branch`).
+3. Make your changes.
+4. Commit your changes (`git commit -am 'Add new feature'`).
+5. Push to the branch (`git push origin my-feature-branch`).
+6. Create a new Pull Request.
+
+Please ensure your code adheres to the project's coding standards and includes appropriate tests.

data/ext/lz4_flex_ext/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "lz4_flex_ext"
+version = "0.1.0"
+edition = "2021"
+authors = ["Ian Ker-Seymer <ian.kerseymer@shopify.com>"]
+license = "MIT"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+lz4_flex = { version = "0.11.3", default-features = true }
+magnus = { version = "0.7.1", features = ["rb-sys"] }
+postcard = "1.0.8"
+rb-sys = { version = "0.9.101", features = ["stable-api-compiled-fallback"] }
+serde = { version = "1.0.208", features = ["derive"] }
+static_assertions = "1.1.0"

data/ext/lz4_flex_ext/extconf.rb

@@ -0,0 +1,7 @@
+# typed: false
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("lz4_flex/lz4_flex_ext")

data/ext/lz4_flex_ext/src/block.rs

@@ -0,0 +1,81 @@
+use lz4_flex::block::{compress_into, decompress_into, get_maximum_output_size};
+
+use crate::Header;
+use crate::LockedRString;
+use crate::{decode_error_class, encode_error_class, helpers::*};
+
+use magnus::{prelude::*, Error, RString};
+
+static_assertions::assert_eq_size!(Header, [u8; 8]);
+
+pub(crate) fn compress_with_header(
+    header: &Header,
+    input: LockedRString,
+    varint: bool,
+) -> Result<RString, Error> {
+    let bufsize = get_maximum_output_size(input.len()) + Header::MAX_SERIALIZED_SIZE;
+    let mut output = RStringMut::buf_new(bufsize);
+    output.expand(bufsize);
+
+    let outbuf = output.as_mut_slice();
+
+    let (header_len, outbuf) = if varint {
+        header.write_varint_to(outbuf)
+    } else {
+        header.write_to(outbuf)
+    };
+
+    let outsize = nogvl_if_large(output.capacity(), || {
+        compress_into(input.as_slice(), outbuf)
+    })
+    .map_err(|e| Error::new(encode_error_class(), e.to_string()))?;
+
+    output.set_len(outsize + header_len);
+    Ok(output.into_inner())
+}
+
+/// Encodes a block of data using LZ4 compression, with encoding awareness of the string.
+pub(crate) fn compress(input: LockedRString) -> Result<RString, Error> {
+    let header = Header::new(input.len() as u32, input.encoding()?);
+
+    compress_with_header(&header, input, false)
+}
+
+pub(crate) fn compress_varint(input: LockedRString) -> Result<RString, Error> {
+    let header = Header::new(input.len() as u32, Encoding::Binary);
+
+    compress_with_header(&header, input, true)
+}
+
+pub(crate) fn decompress_with_header(
+    header: &Header,
+    input_slice: &[u8],
+) -> Result<RString, Error> {
+    let mut output = RStringMut::buf_new(header.size as usize);
+    output.expand(header.size as usize);
+
+    nogvl_if_large(output.capacity(), || {
+        decompress_into(input_slice, output.as_mut_slice())
+    })
+    .map_err(|e| Error::new(decode_error_class(), e.to_string()))?;
+
+    output.set_len(header.size as usize);
+    let output = output.into_inner();
+    output.enc_set(header.encoding().encindex())?;
+
+    Ok(output)
+}
+
+pub(crate) fn decompress(input: LockedRString) -> Result<RString, Error> {
+    let input_slice = input.as_slice();
+    let (header, input_slice) = Header::from_bytes(input_slice)?;
+
+    decompress_with_header(&header, input_slice)
+}
+
+pub(crate) fn decompress_varint(input: LockedRString) -> Result<RString, Error> {
+    let input_slice = input.as_slice();
+    let (header, input_slice) = Header::from_varint(input_slice)?;
+
+    decompress_with_header(&header, input_slice)
+}

data/ext/lz4_flex_ext/src/header.rs

@@ -0,0 +1,118 @@
+use magnus::Error;
+use serde::{Deserialize, Serialize};
+
+use crate::{base_error_class, decode_error_class, Encoding};
+
+#[derive(Debug, Serialize, Deserialize)]
+#[repr(transparent)]
+pub(crate) struct VersionAndEncoding(u8);
+
+impl VersionAndEncoding {
+    const fn new(version: u8, encoding: Encoding) -> Self {
+        Self((version << 4) | encoding as u8)
+    }
+
+    const fn version(&self) -> u8 {
+        self.0 >> 4
+    }
+
+    const fn encoding(&self) -> Encoding {
+        Encoding::from_u8(self.0 & 0b1111)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[repr(C)]
+pub(crate) struct Header {
+    pub(crate) metadata: VersionAndEncoding,
+    pub(crate) size: u32,
+}
+
+impl Header {
+    pub(crate) const MAX_SERIALIZED_SIZE: usize = size_of::<Self>();
+
+    pub(crate) fn from_varint(bytes: &[u8]) -> Result<(Self, &[u8]), Error> {
+        let mut val = 0u32;
+        let mut varbyte_len = 0;
+
+        for &b in bytes.iter() {
+            val |= ((b & 0x7F) as u32) << (7 * varbyte_len);
+            varbyte_len += 1;
+
+            if b & 0x80 == 0 {
+                let header = Self {
+                    metadata: VersionAndEncoding::new(0, Encoding::Binary),
+                    size: val,
+                };
+
+                return Ok((header, &bytes[varbyte_len..]));
+            }
+
+            if varbyte_len > 5 {
+                return Err(Error::new(decode_error_class(), "varint too long"));
+            }
+        }
+
+        Err(Error::new(decode_error_class(), "unexpected end of varint"))
+    }
+
+    pub(crate) fn new(size: u32, encoding: Encoding) -> Self {
+        Self {
+            metadata: VersionAndEncoding::new(1, encoding),
+            size,
+        }
+    }
+
+    pub(crate) const fn encoding(&self) -> Encoding {
+        self.metadata.encoding()
+    }
+
+    pub(crate) fn from_bytes(bytes: &[u8]) -> Result<(Self, &[u8]), Error> {
+        let (header, rest) = postcard::take_from_bytes::<Self>(bytes).map_err(|e| {
+            Error::new(
+                decode_error_class(),
+                format!("failed to deserialize header: {}", e),
+            )
+        })?;
+
+        if header.metadata.version() != 1 {
+            return Err(Error::new(
+                base_error_class(),
+                format!("invalid header version: {}", header.metadata.version()),
+            ));
+        }
+
+        Ok((header, rest))
+    }
+
+    pub(crate) fn write_to<'a>(&self, buf: &'a mut [u8]) -> (usize, &'a mut [u8]) {
+        let serialized_slice_len = {
+            postcard::to_slice(self, buf)
+                .expect("failed to serialize header")
+                .len()
+        };
+
+        (serialized_slice_len, &mut buf[serialized_slice_len..])
+    }
+
+    pub(crate) fn write_varint_to<'a>(&self, buf: &'a mut [u8]) -> (usize, &'a mut [u8]) {
+        let mut val = self.size;
+        let mut index = 0;
+
+        loop {
+            let byte = (val & 0x7f) as u8;
+            val >>= 7;
+
+            if val == 0 {
+                buf[index] = byte;
+                index += 1;
+                break;
+            } else {
+                buf[index] = byte | 0x80;
+                index += 1;
+            }
+        }
+
+        (index, &mut buf[index..])
+    }
+}

data/ext/lz4_flex_ext/src/helpers.rs

@@ -0,0 +1,213 @@
+use serde::{Deserialize, Serialize};
+use std::{cell::Cell, ffi::c_void, marker::PhantomData, mem::MaybeUninit, ptr::null_mut};
+
+use magnus::{
+    encoding::{EncodingCapable, Index},
+    exception::type_error,
+    rb_sys::AsRawValue,
+    value::ReprValue,
+    Error, RString, Ruby, TryConvert, Value,
+};
+use rb_sys::{
+    rb_gc_guard, rb_str_locktmp, rb_str_modify_expand, rb_str_set_len, rb_str_unlocktmp,
+    rb_thread_call_without_gvl, RSTRING_PTR,
+};
+
+use crate::base_error_class;
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+#[repr(u8)]
+pub(crate) enum Encoding {
+    Utf8 = 0,
+    Binary = 1,
+    UsAscii = 2,
+}
+
+impl Encoding {
+    pub(crate) fn encindex(self) -> Index {
+        let ruby = unsafe { Ruby::get_unchecked() };
+
+        match self {
+            Self::Utf8 => ruby.utf8_encindex(),
+            Self::Binary => ruby.ascii8bit_encindex(),
+            Self::UsAscii => ruby.usascii_encindex(),
+        }
+    }
+
+    pub(crate) const fn from_u8(value: u8) -> Self {
+        match value {
+            0 => Self::Utf8,
+            1 => Self::Binary,
+            2 => Self::UsAscii,
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl TryFrom<RString> for Encoding {
+    type Error = Error;
+
+    fn try_from(string: RString) -> Result<Self, Self::Error> {
+        let ruby = unsafe { Ruby::get_unchecked() };
+
+        let enc = string.enc_get();
+
+        if enc == ruby.utf8_encindex() {
+            return Ok(Encoding::Utf8);
+        }
+
+        if enc == ruby.ascii8bit_encindex() {
+            return Ok(Encoding::Binary);
+        }
+
+        if enc == ruby.usascii_encindex() {
+            return Ok(Encoding::UsAscii);
+        }
+
+        Err(Error::new(
+            base_error_class(),
+            "unsupported encoding for string, please use Lz4Flex.compress_block instead".to_owned(),
+        ))
+    }
+}
+
+#[derive(Debug)]
+pub struct LockedRString<'a>(pub(crate) RString, PhantomData<&'a ()>);
+
+impl<'a> LockedRString<'a> {
+    pub(crate) fn new(string: RString) -> Self {
+        unsafe { rb_str_locktmp(string.as_raw()) };
+
+        Self(string, PhantomData)
+    }
+
+    pub(crate) fn as_slice(&self) -> &'a [u8] {
+        unsafe {
+            std::slice::from_raw_parts(RSTRING_PTR(self.0.as_raw()) as *const u8, self.0.len())
+        }
+    }
+
+    pub(crate) fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    pub(crate) fn encoding(&self) -> Result<Encoding, Error> {
+        Encoding::try_from(self.0)
+    }
+}
+
+impl Drop for LockedRString<'_> {
+    fn drop(&mut self) {
+        unsafe { rb_str_unlocktmp(self.0.as_raw()) };
+        // Keep the value alive while we hold this RString
+        let _ = rb_gc_guard!(self.0.as_raw());
+    }
+}
+
+impl TryConvert for LockedRString<'_> {
+    fn try_convert(val: Value) -> Result<Self, magnus::Error> {
+        let rstring: RString = RString::from_value(val).ok_or(magnus::Error::new(
+            type_error(),
+            format!("expected String, got {}", val.class()),
+        ))?;
+        Ok(Self::new(rstring))
+    }
+}
+
+#[derive(Debug)]
+pub struct RStringMut<'a> {
+    inner: RString,
+    capa: Cell<usize>,
+    // Used to safely borrow the underlying rstring buffer by associating it with the lifetime of
+    // the RStringMut
+    _lifetime: PhantomData<&'a ()>,
+}
+
+impl<'a> RStringMut<'a> {
+    pub(crate) fn buf_new(size: usize) -> Self {
+        let string = RString::buf_new(size);
+        Self {
+            inner: string,
+            capa: Cell::new(size),
+            _lifetime: PhantomData,
+        }
+    }
+
+    pub(crate) fn as_mut_slice(&mut self) -> &'a mut [u8] {
+        let raw_value = self.inner.as_raw();
+
+        unsafe {
+            std::slice::from_raw_parts_mut(RSTRING_PTR(raw_value) as *mut u8, self.capa.get())
+        }
+    }
+
+    pub(crate) fn capacity(&self) -> usize {
+        self.capa.get()
+    }
+
+    pub(crate) fn expand(&self, size: usize) {
+        if size <= self.capacity() {
+            return;
+        }
+
+        unsafe { rb_str_modify_expand(self.inner.as_raw(), size as _) };
+    }
+
+    pub(crate) fn set_len(&self, len: usize) {
+        unsafe { rb_str_set_len(self.inner.as_raw(), len as _) }
+    }
+
+    pub(crate) fn into_inner(self) -> RString {
+        self.inner
+    }
+}
+
+impl Drop for RStringMut<'_> {
+    fn drop(&mut self) {
+        // Keep the value alive while we hold this RString
+        let _ = rb_gc_guard!(self.inner.as_raw());
+    }
+}
+
+pub(crate) fn nogvl_if_large<F, R>(input_len: usize, mut func: F) -> R
+where
+    F: FnMut() -> R,
+    R: Sized,
+{
+    const MAX_INPUT_LEN: usize = 8192;
+
+    if input_len > MAX_INPUT_LEN {
+        nogvl(func)
+    } else {
+        func()
+    }
+}
+
+// https://docs.rs/rb-sys/0.9.97/rb_sys/bindings/uncategorized/fn.rb_thread_call_without_gvl.html
+pub(crate) fn nogvl<F, R>(mut func: F) -> R
+where
+    F: FnMut() -> R,
+    R: Sized,
+{
+    unsafe extern "C" fn call_without_gvl<F, R>(arg: *mut c_void) -> *mut c_void
+    where
+        F: FnMut() -> R,
+        R: Sized,
+    {
+        let arg = arg as *mut (&mut F, &mut MaybeUninit<R>);
+        // Safety: we know that the pointer is valid since we initialized it in nogvl
+        let (func, result) = unsafe { &mut *arg };
+        result.write(func());
+
+        null_mut()
+    }
+
+    let result = MaybeUninit::uninit(); //location in memory that has not yet been initalized (init'd in call_without_gvl)
+    let arg_ptr = &(&mut func, &result) as *const _ as *mut c_void; //tuple of func and result (raw + mutable) pointers
+
+    unsafe {
+        // https://github.com/ruby/ruby/blob/ruby_3_3/thread.c#L1550-L1634
+        rb_thread_call_without_gvl(Some(call_without_gvl::<F, R>), arg_ptr, None, null_mut());
+        result.assume_init()
+    }
+}

data/ext/lz4_flex_ext/src/lib.rs

@@ -0,0 +1,71 @@
+mod block;
+mod header;
+mod helpers;
+
+pub(crate) use header::*;
+pub(crate) use helpers::*;
+
+use magnus::{
+    exception::standard_error,
+    function,
+    prelude::*,
+    value::{InnerValue, Lazy},
+    Error, ExceptionClass, RModule, Ruby,
+};
+use rb_sys::ruby_abi_version;
+
+static MODULE_ROOT: Lazy<RModule> = Lazy::new(|ruby| ruby.define_module("Lz4Flex").unwrap());
+
+pub(crate) fn base_error_class() -> ExceptionClass {
+    static BASE_ERROR_CLASS: Lazy<ExceptionClass> = Lazy::new(|ruby| {
+        MODULE_ROOT
+            .get_inner_with(ruby)
+            .define_error("Error", standard_error())
+            .unwrap()
+    });
+    unsafe { BASE_ERROR_CLASS.get_inner_with(&Ruby::get_unchecked()) }
+}
+
+pub(crate) fn encode_error_class() -> ExceptionClass {
+    static ENCODE_ERROR_CLASS: Lazy<ExceptionClass> = Lazy::new(|ruby| {
+        MODULE_ROOT
+            .get_inner_with(ruby)
+            .define_error("EncodeError", base_error_class())
+            .unwrap()
+    });
+    unsafe { ENCODE_ERROR_CLASS.get_inner_with(&Ruby::get_unchecked()) }
+}
+
+pub(crate) fn decode_error_class() -> ExceptionClass {
+    static DECODE_ERROR_CLASS: Lazy<ExceptionClass> = Lazy::new(|ruby| {
+        MODULE_ROOT
+            .get_inner_with(ruby)
+            .define_error("DecodeError", base_error_class())
+            .unwrap()
+    });
+    unsafe { DECODE_ERROR_CLASS.get_inner_with(&Ruby::get_unchecked()) }
+}
+
+ruby_abi_version!();
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let module = MODULE_ROOT.get_inner_with(ruby);
+    let _ = decode_error_class();
+    let _ = encode_error_class();
+
+    module.define_singleton_method("compress", function!(block::compress, 1))?;
+    module.define_singleton_method("decompress", function!(block::decompress, 1))?;
+    module
+        .singleton_class()?
+        .define_alias("deflate", "compress")?;
+    module
+        .singleton_class()?
+        .define_alias("inflate", "decompress")?;
+
+    let varint_module = module.define_module("VarInt")?;
+    varint_module.define_singleton_method("compress", function!(block::compress_varint, 1))?;
+    varint_module.define_singleton_method("decompress", function!(block::decompress_varint, 1))?;
+
+    Ok(())
+}

data/lib/lz4_flex/version.rb

@@ -0,0 +1,6 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Lz4Flex
+  VERSION = "0.1.1.pre2"
+end

data/lib/lz4_flex.rb

@@ -0,0 +1,10 @@
+# typed: strict
+# frozen_string_literal: true
+
+# Tries to require the precompiled extension for the given Ruby version first
+begin
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require "lz4_flex/#{Regexp.last_match(1)}/lz4_flex_ext"
+rescue LoadError
+  require_relative "lz4_flex/lz4_flex_ext"
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: lz4_flex
+version: !ruby/object:Gem::Version
+  version: 0.1.1.pre2
+platform: ruby
+authors:
+- Shopify Engineering
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2024-08-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.101
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.101
+description: A modern LZ4 compression library for Ruby, leveraging the power of the
+  [`lz4_flex`](https://github.com/PSeitz/lz4_flex) Rust crate. This library provides
+  a pure Rust implementation of the LZ4 algorithm, ensuring high performance and safety.
+email:
+- gems@shopify.com
+executables: []
+extensions:
+- ext/lz4_flex_ext/extconf.rb
+extra_rdoc_files: []
+files:
+- Cargo.lock
+- Cargo.toml
+- LICENSE.md
+- README.md
+- ext/lz4_flex_ext/Cargo.toml
+- ext/lz4_flex_ext/extconf.rb
+- ext/lz4_flex_ext/src/block.rs
+- ext/lz4_flex_ext/src/header.rs
+- ext/lz4_flex_ext/src/helpers.rs
+- ext/lz4_flex_ext/src/lib.rs
+- lib/lz4_flex.rb
+- lib/lz4_flex/version.rb
+homepage: https://github.com/Shopify/lz4-flex-rb
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/Shopify/lz4-flex-rb
+  source_code_uri: https://github.com/Shopify/lz4-flex-rb
+  changelog_uri: https://github.com/Shopify/lz4-flex-rb/releases
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.3
+signing_key: 
+specification_version: 4
+summary: A modern LZ4 compression library for Ruby, leveraging the `lz4_flex` Rust
+  crate.
+test_files: []