omnizip 0.4.2 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48ed1a8b9fa1d9c51d338c611430fbd78f12de0a9d5d9de3ebff23ac3b9b767c
-  data.tar.gz: 6b173fab258e290030227c869b703631023c621699e810753259b14ef613ef34
+  metadata.gz: 8cf4c62c7304ef792b48b96486685707c68010f0cc6529b6a7504454b94ea025
+  data.tar.gz: 7c0d5e62fc21a99ccc85520ee0cd3ea32afaae39e76f10826dd56697ffa8376b
 SHA512:
-  metadata.gz: 07fba2b58cca8f0c0429042e4a2813d5618158cb8231625ad1e6f5c32d1bc532d45a8bb952a3087d6274862647dbab0480aca8c96561f24c5a3debf2a1f7dd63
-  data.tar.gz: 8422215bde30fa926b235b20dddbeba195573a82d1678ea98d541be6e37e6e14c7a6ae444e53b1a4505733a6420a4d9d3a53cec3605c751c5bf6eac49678ef86
+  metadata.gz: 539124e9189208423daacdac7cad6735f4a1069358a6ee7e39bb09155524589a2b1a01a6b1c41af6b650648dee327b2c4d36e71ccbbba5ff5e4453168f6ec9ef
+  data.tar.gz: 02ae0fdb9844f006e5289d8bd351b4ac023e8fafd704bd6fc96b6a0489bc27c96d3e7d3e32ef06011a8123b37573d877191e93cdbaa7c0f6e63e139e8485c09e

data/CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Rust tier wiring for every accelerated codec: zstd, deflate,
+  deflate64 (both zlib-framed), gzip, xz, lzip, and lzma-alone now
+  route decode through the Rust backend when it loads, each pinned by
+  a cross-implementation differential. A Rust failure falls back to
+  the Ruby core — auto mode is never worse than pure Ruby.
+
 ### Fixed
 - RAR recovery-record repair fails loudly instead of pretending:
   `recover_with_reed_solomon` returned nil and `recover_with_xor`

data/lib/omnizip/algorithms/deflate.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "stringio"
+
 # Copyright (C) 2025 Ribose Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
@@ -66,8 +68,13 @@ module Omnizip
       # @return [void]
       def compress(input_stream, output_stream, options = nil)
         input_data = input_stream.read
-        encoder = Encoder.new(output_stream, build_encoder_options(options))
-        encoder.encode_stream(input_data)
+        level = build_encoder_options(options)[:level] || 6
+        output_stream.write(
+          Backends.compress("zlib", input_data, level) do
+            encoder = Encoder.new(output_stream, build_encoder_options(options))
+            encoder.encode_stream(input_data)
+          end,
+        )
       end
 
       # Decompress Deflate-compressed data
@@ -78,8 +85,12 @@ module Omnizip
       # @return [void]
       def decompress(input_stream, output_stream, _options = nil)
         output_stream.set_encoding(Encoding::BINARY)
-        decoder = Decoder.new(input_stream)
-        decompressed = decoder.decode_stream
+        compressed = input_stream.read
+        # The encoder's streams carry the zlib wrapper (RFC 1950) —
+        # the Rust "zlib" name speaks exactly that framing.
+        decompressed = Backends.decompress("zlib", compressed, Backends::UNKNOWN_LENGTH) do
+          Decoder.new(StringIO.new(compressed)).decode_stream
+        end
         output_stream.write(decompressed)
       end
 

data/lib/omnizip/algorithms/deflate64.rb

@@ -42,21 +42,22 @@ module Omnizip
       # @param options [Hash] Compression options
       # @option options [Integer] :level Compression level (1-9)
       def compress(input, output, options = {})
-        level = options[:level] || Zlib::DEFAULT_COMPRESSION
-
+        raw_level = options[:level] || Zlib::DEFAULT_COMPRESSION
         data = input.read
         return if data.nil? || data.empty?
 
-        # Use Zlib::Deflate with maximum window size
-        deflater = Zlib::Deflate.new(
-          level,
-          Zlib::MAX_WBITS, # Maximum window size
-          Zlib::MAX_MEM_LEVEL,
-        )
-
-        compressed = deflater.deflate(data, Zlib::FINISH)
-        deflater.close
-
+        # Zlib::DEFAULT_COMPRESSION (-1) maps to the FFI's 6.
+        level = raw_level.negative? ? 6 : raw_level.clamp(0, 9)
+        compressed = Backends.compress("zlib", data, level) do
+          deflater = Zlib::Deflate.new(
+            raw_level,
+            Zlib::MAX_WBITS,
+            Zlib::MAX_MEM_LEVEL,
+          )
+          result = deflater.deflate(data, Zlib::FINISH)
+          deflater.close
+          result
+        end
         output.write(compressed)
       end
 
@@ -73,10 +74,15 @@ module Omnizip
         output.set_encoding(Encoding::BINARY)
         output.binmode
 
-        # Use Zlib::Inflate with maximum window size
-        inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
-        decompressed = inflater.inflate(compressed)
-        inflater.close
+        # The streams this algorithm produces/consumes are zlib
+        # containers (RFC 1950); the Rust "zlib" name speaks exactly
+        # that, so the tier swap is byte-transparent.
+        decompressed = Backends.decompress("zlib", compressed, Backends::UNKNOWN_LENGTH) do
+          inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
+          result = inflater.inflate(compressed)
+          inflater.close
+          result
+        end
 
         # Force binary encoding to match original data
         decompressed.force_encoding(Encoding::BINARY)

data/lib/omnizip/algorithms/zstandard.rb

@@ -86,8 +86,13 @@ module Omnizip
       # @return [void]
       def compress(input_stream, output_stream, options = nil)
         input_data = input_stream.read
-        encoder = Encoder.new(output_stream, build_encoder_options(options))
-        encoder.encode_stream(input_data)
+        level = build_encoder_options(options)[:level] || 3
+        output_stream.write(
+          Backends.compress("zstd", input_data, level) do
+            encoder = Encoder.new(output_stream, build_encoder_options(options))
+            encoder.encode_stream(input_data)
+          end,
+        )
       end
 
       # Decompress Zstandard-compressed data
@@ -98,8 +103,10 @@ module Omnizip
       # @return [void]
       def decompress(input_stream, output_stream, _options = nil)
         output_stream.set_encoding(Encoding::BINARY)
-        decoder = Decoder.new(input_stream)
-        decompressed = decoder.decode_stream
+        compressed = input_stream.read
+        decompressed = Backends.decompress("zstd", compressed, Backends::UNKNOWN_LENGTH) do
+          Decoder.new(StringIO.new(compressed)).decode_stream
+        end
         output_stream.write(decompressed)
       end
 

data/lib/omnizip/backends.rb

@@ -27,19 +27,23 @@ module Omnizip
   #     over frame stability.
   module Backends
     # Codecs whose Rust encoder is BYTE-IDENTICAL to the pure-Ruby
-    # encoder (empirical, differential-corpus-proven). 2026-09-19
-    # probe: bzip2 L6 on a 50 KB corpus => Ruby 906 B vs Rust 874 B —
-    # DIFFERENT valid streams. No codec qualifies; encode stays
-    # pure-Ruby under :auto so downstream content addressing never
-    # sees tier-dependent bytes. Force with OMNIZIP_BACKEND=rust.
-    RUST_ENCODE_IDENTICAL = {}.freeze
+    # 2026-09-20 policy flip (owner directive): RUST IS THE AUTHORITY
+    # — its encoders/decoders are the ports validated against the
+    # reference C/C++ implementations (bzip2, xz/liblzma, zstd,
+    # libdeflate), and they are determinism-contracted (same input +
+    # level => same bytes on every machine). :auto therefore routes
+    # BOTH directions through Rust whenever the cdylib loads; encode
+    # falls back to the Ruby core on any Rust error, so :auto is
+    # never worse than pure Ruby. The pure-Ruby cores remain the
+    # portable fallback and the OMNIZIP_BACKEND=ruby escape hatch.
+    RUST_AUTHORITATIVE = true
 
     # usize::MAX — the FFI's "caller does not know the plaintext
     # size" sentinel (streaming decode contract).
     UNKNOWN_LENGTH = 0xFFFF_FFFF_FFFF_FFFF
 
     class << self
-      def for(codec, direction)
+      def for(codec, _direction)
         mode = ENV.fetch("OMNIZIP_BACKEND", "auto")
 
         return backend_rust if mode == "rust"
@@ -48,26 +52,46 @@ module Omnizip
         return backend_ruby unless Implementations::Rust::Library.available?
         return backend_ruby unless Implementations::Rust::Library::CODECS.include?(codec)
 
-        direction == :encode && !RUST_ENCODE_IDENTICAL.fetch(codec, false) ? backend_ruby : backend_rust
+        backend_rust
       end
 
       # Compress through the tier-selected backend. The block is
       # the pure-Ruby core — evaluated ONLY on the Ruby path, so
-      # the accelerated path never pays for it.
+      # the accelerated path never pays for it. A Rust encode error
+      # falls back to the Ruby core (auto is never worse than pure
+      # Ruby); only the forced `rust` mode propagates.
       def compress(codec, data, level, &ruby_core)
         backend = Backends.for(codec, :encode)
         return yield if backend == RubyBackend
 
-        backend.compress(codec, data, level)
+        begin
+          backend.compress(codec, data, level)
+        rescue Implementations::Rust::Error => e
+          raise if ENV.fetch("OMNIZIP_BACKEND", "auto") == "rust"
+
+          warn "omnizip: rust #{codec} encode failed (#{e.message}); using ruby" if ENV["OMNIZIP_BACKEND_DEBUG"]
+          yield
+        end
       end
 
       # Decompress through the tier-selected backend (output-
-      # invariant: :auto prefers Rust whenever loadable).
+      # invariant: :auto prefers Rust whenever loadable). A Rust
+      # failure falls back to the Ruby core — the Ruby readers are
+      # the tolerant superset (legacy container variants, malformed
+      # inputs they historically accept), so :auto must never be
+      # WORSE than pure Ruby. Only the forced `rust` mode propagates.
       def decompress(codec, data, expected_len, &ruby_core)
         backend = Backends.for(codec, :decode)
         return yield if backend == RubyBackend
 
-        backend.decompress(codec, data, expected_len)
+        begin
+          backend.decompress(codec, data, expected_len)
+        rescue Implementations::Rust::Error => e
+          raise if ENV.fetch("OMNIZIP_BACKEND", "auto") == "rust"
+
+          warn "omnizip: rust #{codec} decode failed (#{e.message}); using ruby" if ENV["OMNIZIP_BACKEND_DEBUG"]
+          yield
+        end
       end
 
       private

data/lib/omnizip/formats/gzip.rb

@@ -91,18 +91,15 @@ module Omnizip
           # Write GZIP header
           write_header(output_io, mtime, original_name)
 
-          # Compress data with Deflate
-          deflate = Zlib::Deflate.new(
-            level,
-            Zlib::MAX_WBITS + 16, # Use GZIP wrapper
-          )
-          compressed = deflate.deflate(input_data, Zlib::FINISH)
+          # RFC 1952 member: header + RAW deflate + CRC32/ISIZE footer.
+          # The old implementation layered zlib's own GZIP wrapper
+          # under this header (a second 1F 8B magic) — the gem's
+          # reader tolerated it, but every standard gzip tool
+          # rejected the output. Raw-window mode emits bare DEFLATE.
+          deflate = Zlib::Deflate.new(level, -Zlib::MAX_WBITS)
+          output_io.write(deflate.deflate(input_data, Zlib::FINISH))
           deflate.close
 
-          # Extract just the compressed data (remove zlib's GZIP wrapper)
-          # We'll write our own footer
-          output_io.write(compressed[0...-8]) if compressed.bytesize > 8
-
           # Write GZIP footer
           write_footer(output_io, input_data)
         end
@@ -113,15 +110,27 @@ module Omnizip
         # @param output_io [IO] Output stream
         # @return [Hash] Metadata (original_name, mtime)
         def decompress_stream(input_io, output_io)
-          # Read and parse header
-          metadata = read_header(input_io)
-
-          # Decompress with Deflate
-          inflate = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
-          decompressed = inflate.inflate(input_io.read)
-          inflate.close
+          data = input_io.read
+
+          # Metadata comes from the header either way (cheap parse).
+          metadata = read_header(StringIO.new(data))
+
+          # The tier gets the WHOLE member (header + deflate + trailer)
+          # — the Rust "gzip" name decodes the container end-to-end.
+          decompressed = Backends.decompress("gzip", data, Backends::UNKNOWN_LENGTH) do
+            body = StringIO.new(data)
+            read_header(body) # skip past the header
+            # Legacy double-header members (written before the RFC
+            # 1952 fix) carry zlib's own 10-byte GZIP header here;
+            # standard members carry bare DEFLATE.
+            rest = body.read
+            rest = rest.byteslice(10..) if rest.byteslice(0, 2) == "\x1F\x8B".b
+            inflate = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+            result = inflate.inflate(rest)
+            inflate.close
+            result
+          end
 
-          # Write decompressed data
           output_io.write(decompressed)
 
           metadata

data/lib/omnizip/formats/xz.rb

@@ -89,11 +89,17 @@ module Omnizip
 
           case format
           when :lz
-            decode_lzip(data)
+            Backends.decompress("lzip", data, Backends::UNKNOWN_LENGTH) do
+              decode_lzip(data)
+            end
           when :lzma_alone
-            decode_lzma_alone(data)
+            Backends.decompress("lzma-alone", data, Backends::UNKNOWN_LENGTH) do
+              decode_lzma_alone(data)
+            end
           when :xz
-            decode_xz_stream(data)
+            Backends.decompress("xz", data, Backends::UNKNOWN_LENGTH) do
+              decode_xz_stream(data)
+            end
           else
             raise FormatError, "Unknown LZMA format: cannot detect valid format"
           end

data/lib/omnizip/implementations/rust/library.rb

@@ -16,7 +16,13 @@ module Omnizip
       # built; compress/decompress are pure C calls).
       class Library
         # Codecs the cdylib dispatches by name.
-        CODECS = %w[bzip2 zstd lzma xz].freeze
+        # Names the cdylib dispatches: the codec-level acceleration
+        # surface (both directions by default — Rust is the
+        # authority; Ruby is the fallback).
+        CODECS = %w[
+          bzip2 zstd lzma xz lzma-alone lzip
+          deflate deflate64 zlib gzip
+        ].freeze
 
         class << self
           # The shared handle, or nil when the cdylib cannot load.
@@ -36,7 +42,10 @@ module Omnizip
           # specs exercising fallback/absence paths need a clean
           # slate; production never calls this).
           def forget!
-            @instance = nil if defined?(@instance)
+            # Remove the ivar entirely: assigning nil would leave it
+            # defined, and `instance`'s `defined?` guard would return
+            # the nil forever without ever re-opening the library.
+            remove_instance_variable(:@instance) if defined?(@instance)
             nil
           end
 

data/lib/omnizip/implementations/rust.rb

@@ -13,11 +13,11 @@ module Omnizip
     #   1. ENV['OMNIZIP_FFI_DYLIB'] (explicit path)
     #   2. ../target/release/ inside a sibling omnizip-rs checkout
     #
-    # Tier policy (see Omnizip::Backends): decode is always
-    # output-invariant, so `auto` uses Rust for decode whenever
-    # available; encode uses Rust only when byte-identity with the
-    # pure-Ruby encoder has been verified for that codec
-    # (RUST_ENCODE_IDENTICAL), keeping content addressing stable.
+    # Tier policy (see Omnizip::Backends): Rust is the authority —
+    # validated against the reference C/C++ implementations and
+    # determinism-contracted — so `auto` routes BOTH directions
+    # through Rust whenever the library loads, with Ruby-core
+    # fallback on any Rust error.
     module Rust
       autoload :Library, "omnizip/implementations/rust/library"
     end

data/lib/omnizip/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Omnizip
-  VERSION = "0.4.2"
+  VERSION = "0.4.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omnizip
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.4
 platform: ruby
 authors:
 - Ribose Inc.