omq-lz4 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ade7fc716054707c2e05bade867b01333708de30df98e3acb524f8da1a7d98d4
-  data.tar.gz: 7ae5fade7f0d88eed7f6f7f8875373a044127098d596b9f6bae5554d585b1e90
+  metadata.gz: 4217a939f757a07b0e3e2ad1a905d11ea3cd017e54baa2f059a75f304386bdab
+  data.tar.gz: 1d0f4e4076e0cf02291b3f714fe20c721b8e05e0d4d17af4fbc8662e83897319
 SHA512:
-  metadata.gz: 59afc48b58c8c1efac0973bffa702cac3958b0150b4eb51fe9dec82dacc1c9735446d83e58ac6963f0302e13b26a0890b1eaeff5c0d7d846830c561775a848f4
-  data.tar.gz: 17c66cce9f79f1a375a8614e3aa3cc45071277432521ec96d446daede230b1e6e1a71003b88062345c48550c128e524aa10160d6a06c41e6c7597309e138d5f0
+  metadata.gz: a963f09065ef7a019a8b766a00093e5ef84ba9489fed66d3524da5e8a0d8d956d27cd01534926ac1cb3f5c5088439da9e0db5c5f71c0b7b58b7b90db8ccfc8bf
+  data.tar.gz: 86e7b5fe26c080a13a6b5c5c44b0e0a250bc45d650b2a9f76f4a80330209d6e24374b8a6412dd28118cb971619f56d3bede7a5ed937c91c5a80fd520ab17b5ac

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.3.0 (2026-05-11)
+
+### Added
+
+- **LZ4M multi-block encoding/decoding** (RFC §5.3a, §5.4a, §5.5 rule 4).
+  Parts larger than `LZ4M_BLOCK_SIZE` (1 GiB) are split into independently
+  decodable blocks, each compressed against the installed dict (if any).
+  `encode_part` / `decode_part` accept a `block_size:` keyword for testing
+  with smaller-than-protocol block sizes.
+- `LZ4M_SENTINEL` (`"LZ4M"`) and `LZ4M_BLOCK_SIZE` (1,073,741,824) constants
+  in `OMQ::LZ4::Codec`.
+- LZ4B `decompressed_size` cap: the decoder now rejects single-block parts
+  whose declared `decompressed_size` exceeds `LZ4M_BLOCK_SIZE` (RFC §5.5
+  rule 3).
+- Codec tests for LZ4M round-trips (with and without dict, partial last
+  block, random bytes), malformed LZ4M inputs (truncated, leftover bytes,
+  corrupt block data, budget overrun), and the LZ4B block size limit.
+
 ## 0.2.0 (2026-05-04)
 
 ### Changed

data/README.md

@@ -4,7 +4,6 @@
 [![License: ISC](https://img.shields.io/badge/License-ISC-blue.svg)](LICENSE)
 [![Ruby](https://img.shields.io/badge/Ruby-%3E%3D%203.3-CC342D?logo=ruby&logoColor=white)](https://www.ruby-lang.org)
 
-> **Status:** 0.1.0 — first landable release. See
 > [RFC.md](RFC.md) for the wire-format spec and
 > [CHANGELOG.md](CHANGELOG.md) for what's in.
 

data/lib/omq/lz4/codec.rb

@@ -14,18 +14,22 @@ module OMQ
     # Each wire part begins with a 4-byte sentinel:
     #
     #   00 00 00 00   uncompressed plaintext
-    #   4C 5A 34 42   LZ4-compressed block ("LZ4B" in ASCII)
+    #   4C 5A 34 42   LZ4-compressed single block ("LZ4B" in ASCII)
+    #   4C 5A 34 4D   LZ4-compressed multi-block ("LZ4M" in ASCII)
     #   4C 5A 34 44   dictionary shipment ("LZ4D" in ASCII)
     #
-    # `decode_part` handles UNCOMPRESSED and LZ4B only. Dictionary
+    # `decode_part` handles UNCOMPRESSED, LZ4B, and LZ4M. Dictionary
     # shipments are a transport-layer concern: the transport peeks the
     # first 4 bytes of each incoming wire part, routes LZ4D to
     # `decode_dict_shipment`, and never hands a shipment to `decode_part`.
     module Codec
       UNCOMPRESSED_SENTINEL = "\x00\x00\x00\x00".b.freeze
       LZ4B_SENTINEL         = "LZ4B".b.freeze
+      LZ4M_SENTINEL         = "LZ4M".b.freeze
       LZ4D_SENTINEL         = "LZ4D".b.freeze
 
+      LZ4M_BLOCK_SIZE = 1_073_741_824
+
       # Size thresholds below which compression isn't worth attempting.
       # Empirically tuned on Lorem-ipsum-like input via
       # bench/min_compress_size_sweep.rb: for block-format LZ4 the
@@ -65,10 +69,11 @@ module OMQ
       # `min_size` overrides the default threshold. Nil (the default)
       # picks `MIN_COMPRESS_NO_DICT` for a no-dict codec and
       # `MIN_COMPRESS_WITH_DICT` for a dict codec.
-      def encode_part(plaintext, block_codec:, min_size: nil)
+      def encode_part(plaintext, block_codec:, min_size: nil, block_size: LZ4M_BLOCK_SIZE)
         min_size ||= block_codec.has_dict? ? MIN_COMPRESS_WITH_DICT : MIN_COMPRESS_NO_DICT
 
         return encode_passthrough(plaintext) if plaintext.bytesize < min_size
+        return encode_multi_block(plaintext, block_codec, block_size) if plaintext.bytesize > block_size
 
         compressed = block_codec.compress(plaintext)
 
@@ -91,7 +96,7 @@ module OMQ
       #
       # Does not handle LZ4D shipments; transport must route those to
       # `decode_dict_shipment` before calling here.
-      def decode_part(wire_bytes, block_codec:, max_size: nil)
+      def decode_part(wire_bytes, block_codec:, max_size: nil, block_size: LZ4M_BLOCK_SIZE)
         if wire_bytes.bytesize < 4
           raise ProtocolError, "wire part too short (< 4 bytes)"
         end
@@ -107,6 +112,10 @@ module OMQ
             raise ProtocolError, "LZ4B part too short (< 12 bytes, no room for size field)"
           end
           decompressed_size = wire_bytes.byteslice(4, 8).unpack1("Q<")
+          if decompressed_size > block_size
+            raise ProtocolError,
+              "LZ4B decompressed_size #{decompressed_size} exceeds block size limit #{block_size}"
+          end
           check_size!(decompressed_size, max_size)
           block = wire_bytes.byteslice(12, wire_bytes.bytesize - 12)
           begin
@@ -114,8 +123,9 @@ module OMQ
           rescue RLZ4::DecompressError => e
             raise ProtocolError, "LZ4B decode failed: #{e.message}"
           end
+        when LZ4M_SENTINEL
+          decode_multi_block(wire_bytes, block_codec, max_size, block_size)
         when LZ4D_SENTINEL
-          # Should not reach decode_part; transport should have routed this.
           raise ProtocolError,
             "LZ4D dictionary shipment seen at decode_part (transport should route to decode_dict_shipment)"
         else
@@ -154,6 +164,70 @@ module OMQ
       class << self
         private
 
+        def encode_multi_block(plaintext, block_codec, block_size)
+          buf = String.new(encoding: Encoding::BINARY)
+          buf << LZ4M_SENTINEL
+          buf << [plaintext.bytesize].pack("Q<")
+
+          offset = 0
+          while offset < plaintext.bytesize
+            chunk_size = [block_size, plaintext.bytesize - offset].min
+            chunk = plaintext.byteslice(offset, chunk_size)
+            compressed = block_codec.compress(chunk)
+            buf << [compressed.bytesize].pack("V")
+            buf << compressed
+            offset += chunk_size
+          end
+
+          buf
+        end
+
+
+        def decode_multi_block(wire_bytes, block_codec, max_size, block_size)
+          if wire_bytes.bytesize < 12
+            raise ProtocolError, "LZ4M part too short (< 12 bytes, no room for size field)"
+          end
+
+          decompressed_size = wire_bytes.byteslice(4, 8).unpack1("Q<")
+          check_size!(decompressed_size, max_size)
+
+          output = String.new(capacity: decompressed_size, encoding: Encoding::BINARY)
+          offset = 12
+          remaining = decompressed_size
+
+          while remaining > 0
+            if offset + 4 > wire_bytes.bytesize
+              raise ProtocolError, "LZ4M truncated: no room for block length at offset #{offset}"
+            end
+
+            compressed_len = wire_bytes.byteslice(offset, 4).unpack1("V")
+            offset += 4
+
+            if offset + compressed_len > wire_bytes.bytesize
+              raise ProtocolError, "LZ4M truncated: block at offset #{offset} extends past wire end"
+            end
+
+            block_data = wire_bytes.byteslice(offset, compressed_len)
+            offset += compressed_len
+
+            block_decompressed_size = [block_size, remaining].min
+            begin
+              output << block_codec.decompress(block_data, decompressed_size: block_decompressed_size)
+            rescue RLZ4::DecompressError => e
+              raise ProtocolError, "LZ4M block decode failed: #{e.message}"
+            end
+
+            remaining -= block_decompressed_size
+          end
+
+          if offset != wire_bytes.bytesize
+            raise ProtocolError, "LZ4M: #{wire_bytes.bytesize - offset} leftover bytes after last block"
+          end
+
+          output
+        end
+
+
         def encode_passthrough(plaintext)
           UNCOMPRESSED_SENTINEL + plaintext
         end

data/lib/omq/lz4/version.rb

@@ -2,6 +2,6 @@
 
 module OMQ
   module LZ4
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omq-lz4
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Patrik Wenger