omnizip 0.3.58 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04bb0d1c81ef7f4571014bb6679ad0d4750d4c078729daea82c88f643e246818
-  data.tar.gz: 7523e6a77a43db5eb02ec7c16ef4adf0751150129f329af82eb11297f93a68f6
+  metadata.gz: 06cbc9a2cd22e74a84d82211efa5e000d857da117bae9a995b76520839211362
+  data.tar.gz: e8ab9ec5638953dda59f4b43aa299cfa55262f77d3c9e9f795bbe8800d443592
 SHA512:
-  metadata.gz: da1c027f6e44246e6421760d3197501db38957aeb1541c0db6e3553ab73b5e03a221a362b0165d4196bf1185f0022b341529321cdce9c2a339c3cf2f31b4f733
-  data.tar.gz: 7732ed6d8268d006195f7648980e2f8619ea89d45767932afd587f508592f627697a9cc436f6f3e1a01b41a8a579034f654c1bc33cc3331fced28af5bd68b8ff
+  metadata.gz: ecd9426b7a3f0a75c68821ec6c2c4c0f0f4f6f5e5ae7a252dc6e1e4fb9f5eb94a01b84f937c797581b44584e8cc4ba244d35e37c2338b1401c46de42299a1627
+  data.tar.gz: 268f70fb015d2e21b8079cae5ea4e09a83e036e967cf83bed03cc37de45b61f1875547b1443b419eec502a3bcc264e323d19bb8c503f72437cfb47bbd9d31853

data/CHANGELOG.md

@@ -7,6 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- `rake rust:build` — one command to build and vendor the Rust
+  backend's cdylib (clones omnizip-rs or reuses a checkout via
+  REPO=); the loader gains the `vendor/` resolution tier behind
+  `OMNIZIP_FFI_DYLIB`. No toolchain, no failure: the gem stays
+  pure Ruby. docs/rust-backend.md documents the tier policy,
+  resolution order, and CI pinning.
+
+
+### Added
+- Rust-accelerated backend tier: `Omnizip::Backends` is the single
+  tier switch between the pure-Ruby cores and the `omnizip-ffi`
+  cdylib (stdlib Fiddle — no compiled extension). Under the default
+  `auto` policy decode prefers the Rust decoders (output-invariant
+  and carrying fixes the Ruby cores lack); encode stays pure-Ruby
+  so frame bytes never depend on the tier (content addressing
+  stability). `OMNIZIP_BACKEND=rust|ruby` forces a tier; load
+  failures fall back to pure Ruby silently. A cross-tier
+  differential spec pins decode agreement in both encode
+  directions.
+- BZip2 routes its byte core through the backend seam (both paths
+  share the algorithm class's IO plumbing).
+
+### Fixed
+- Ruby 4.0 compatibility for the tier loader: fiddle moved out of
+  the default gems and is now required lazily with a fallback;
+  library resolution no longer references Fiddle constants when
+  fiddle is absent and considers `.dll` candidates on Windows.
+
+## [0.3.58] - 2026-09-03
+
+### Fixed
+- `solid: true` on the RAR5 writer silently fell back to
+  independent STORE entries (the RAR-compatible LZSS encoder is not
+  available); the writer now warns about the fallback. The
+  creating-archives guide's solid section stops advertising
+  compression ratios the option cannot deliver and the stale
+  `(v0.5.0)` version-gate labels are gone from shipped features.
+- The ZIP writer never emits ZIP64 structures; past the classic
+  limits `Array#pack` failed with a bare `RangeError` that named no
+  limit. Both write paths now raise `UnsupportedFormatError` naming
+  the 65,535-entry / 4 GiB size / 4 GiB offset classic-ZIP limits
+  (ZIP64 writing is not supported).
+
+### Changed
+- The decompress command's archive-vs-stream routing derives from
+  the registry's extension routes instead of a private table, so
+  formats added to the registry route correctly instead of silently
+  falling into the single-file stream path.
+
 ## [0.3.57] - 2026-09-02
 
 ### Fixed

data/lib/omnizip/algorithms/bzip2.rb

@@ -67,7 +67,10 @@ module Omnizip
       # @return [void]
       def compress(input_stream, output_stream, options = nil)
         level = level_from(options)
-        output_stream.write(Bz2.compress(input_stream.read, level))
+        data = input_stream.read
+        output_stream.write(
+          Backends.compress("bzip2", data, level) { Bz2.compress(data, level) },
+        )
       end
 
       # Decompress BZip2-compressed data
@@ -78,7 +81,11 @@ module Omnizip
       # @return [void]
       def decompress(input_stream, output_stream, _options = nil)
         output_stream.set_encoding(Encoding::BINARY)
-        output_stream.write(Bz2.decompress(input_stream.read))
+        compressed = input_stream.read
+        plain = Backends.decompress("bzip2", compressed, Backends::UNKNOWN_LENGTH) do
+          Bz2.decompress(compressed)
+        end
+        output_stream.write(plain)
       end
 
       private

data/lib/omnizip/backends.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Omnizip
+  # Byte-level codec backend resolution — THE tier switch
+  # (TODO.ref-parity/62). Both paths share the algorithm classes'
+  # IO plumbing; only the byte core swaps.
+  #
+  # Policy:
+  #   OMNIZIP_BACKEND=ruby   force pure Ruby everywhere
+  #   OMNIZIP_BACKEND=rust   require the Rust library (raises when absent)
+  #   OMNIZIP_BACKEND=auto   (default) Rust for directions marked
+  #                          safe, Ruby otherwise, whenever the
+  #                          cdylib is loadable
+  #
+  # Safety marks (owner guidance 2026-09-19: some Ruby cores are
+  # LESS accurate than the Rust ports — be careful which tier is
+  # trusted for what):
+  #   - decompress -> :rust by default. Decode is output-invariant
+  #     AND the Rust decoders carry fixes the Ruby cores lack;
+  #     disagreements are treated as Ruby-side accuracy findings
+  #     (see the cross-tier differential spec), not coin flips.
+  #   - compress -> :ruby by default. Byte-stability for content
+  #     addressing (DropId dedup assumes identical frames); the
+  #     Rust encoders produce DIFFERENT valid streams (probed:
+  #     bzip2 L6, Ruby 906 B vs Rust 874 B). OMNIZIP_BACKEND=rust
+  #     opts into Rust encode where correctness-of-ratio is wanted
+  #     over frame stability.
+  module Backends
+    # Codecs whose Rust encoder is BYTE-IDENTICAL to the pure-Ruby
+    # encoder (empirical, differential-corpus-proven). 2026-09-19
+    # probe: bzip2 L6 on a 50 KB corpus => Ruby 906 B vs Rust 874 B —
+    # DIFFERENT valid streams. No codec qualifies; encode stays
+    # pure-Ruby under :auto so downstream content addressing never
+    # sees tier-dependent bytes. Force with OMNIZIP_BACKEND=rust.
+    RUST_ENCODE_IDENTICAL = {}.freeze
+
+    # usize::MAX — the FFI's "caller does not know the plaintext
+    # size" sentinel (streaming decode contract).
+    UNKNOWN_LENGTH = 0xFFFF_FFFF_FFFF_FFFF
+
+    class << self
+      def for(codec, direction)
+        mode = ENV.fetch("OMNIZIP_BACKEND", "auto")
+
+        return backend_rust if mode == "rust"
+        return backend_ruby if mode == "ruby"
+
+        return backend_ruby unless Implementations::Rust::Library.available?
+        return backend_ruby unless Implementations::Rust::Library::CODECS.include?(codec)
+
+        direction == :encode && !RUST_ENCODE_IDENTICAL.fetch(codec, false) ? backend_ruby : backend_rust
+      end
+
+      # Compress through the tier-selected backend. The block is
+      # the pure-Ruby core — evaluated ONLY on the Ruby path, so
+      # the accelerated path never pays for it.
+      def compress(codec, data, level, &ruby_core)
+        backend = Backends.for(codec, :encode)
+        return yield if backend == RubyBackend
+
+        backend.compress(codec, data, level)
+      end
+
+      # Decompress through the tier-selected backend (output-
+      # invariant: :auto prefers Rust whenever loadable).
+      def decompress(codec, data, expected_len, &ruby_core)
+        backend = Backends.for(codec, :decode)
+        return yield if backend == RubyBackend
+
+        backend.decompress(codec, data, expected_len)
+      end
+
+      private
+
+      def backend_rust
+        RustBackend
+      end
+
+      def backend_ruby
+        RubyBackend
+      end
+    end
+
+    # The pure-Ruby path: a no-op backend — callers fall back to
+    # their own in-tree cores, which ARE the reference. Kept as an
+    # object for symmetry with the Rust side.
+    module RubyBackend
+      module_function
+
+      # :nocov:
+      def compress(codec, _data, _level)
+        raise Error, "#{codec}: ruby core not wired through Backends (call it directly)"
+      end
+
+      def decompress(codec, _data, _expected_len)
+        raise Error, "#{codec}: ruby core not wired through Backends (call it directly)"
+      end
+      # :nocov:
+    end
+
+    # The Rust-accelerated path over the Fiddle handle.
+    module RustBackend
+      module_function
+
+      def compress(codec, data, level)
+        Implementations::Rust::Library.instance.compress(codec, data, level)
+      end
+
+      def decompress(codec, data, expected_len)
+        Implementations::Rust::Library.instance.decompress(codec, data, expected_len)
+      end
+    end
+
+    class Error < StandardError; end
+  end
+end

data/lib/omnizip/implementations/rust/library.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+begin
+  # Ruby 4.0 moved fiddle out of the default gems.
+  require "fiddle"
+  require "pathname"
+rescue LoadError
+  # :nocov:
+end
+
+module Omnizip
+  module Implementations
+    module Rust
+      # The loaded cdylib: one handle, lazily opened, thread-safe
+      # after initialization (Fiddle::Function is immutable once
+      # built; compress/decompress are pure C calls).
+      class Library
+        # Codecs the cdylib dispatches by name.
+        CODECS = %w[bzip2 zstd lzma xz].freeze
+
+        class << self
+          # The shared handle, or nil when the cdylib cannot load.
+          # Load failures are cached: a missing library is a stable
+          # condition, not something to re-attempt per call.
+          def instance
+            return @instance if defined?(@instance)
+
+            @instance = open
+          end
+
+          def available?
+            !instance.nil?
+          end
+
+          # Drop the cached handle so resolution reruns (test seam:
+          # specs exercising fallback/absence paths need a clean
+          # slate; production never calls this).
+          def forget!
+            @instance = nil if defined?(@instance)
+            nil
+          end
+
+          # Built lazily: the Fiddle::TYPE_* constants only resolve
+          # after a successful require; environments without fiddle
+          # (Ruby 4.0 without the gem) never touch this table.
+          def function_table
+            {
+              last_error: ["ozip_last_error", [], Fiddle::TYPE_VOIDP],
+              free: ["ozip_free", %i[voidp size_t], Fiddle::TYPE_VOID],
+              compress: [
+                "ozip_compress",
+                %i[voidp voidp size_t int voidp],
+                Fiddle::TYPE_VOIDP,
+              ],
+              decompress: [
+                "ozip_decompress",
+                %i[voidp voidp size_t size_t voidp],
+                Fiddle::TYPE_VOIDP,
+              ],
+            }
+          end
+
+          private
+
+          def open
+            path = resolve_path
+            return nil if path.nil?
+
+            handle = Fiddle.dlopen(path.to_s)
+            funcs = function_table.each_with_object({}) do |(key, (name, args, ret)), h|
+              h[key] = Fiddle::Function.new(handle[name], args, ret)
+            end
+            new(funcs)
+          rescue StandardError => e # includes Fiddle::DLError
+            warn "omnizip: rust backend unavailable (#{e.message}); using pure Ruby" if ENV["OMNIZIP_BACKEND"] == "rust"
+            nil
+          end
+
+          def resolve_path
+            explicit = ENV.fetch("OMNIZIP_FFI_DYLIB", nil)
+            return Pathname.new(explicit) if explicit && File.file?(explicit)
+
+            candidates = [
+              Pathname.new(__dir__).join("../../../../ext/libomnizip_ffi"),
+              sibling_checkout,
+            ].compact.flatten
+
+            %w[.dylib .so .dll].each do |ext|
+              candidates.each do |dir|
+                path = dir.join("libomnizip_ffi#{ext}")
+                return path if path.file?
+              end
+            end
+            nil
+          end
+
+          # A sibling omnizip-rs checkout: ../omnizip-rs relative to
+          # this gem's repository root.
+          def sibling_checkout
+            gem_root = Pathname.new(__dir__).join("../../../../..")
+            %w[omnizip-rs target/release].map do |rel|
+              gem_root.join(rel, "target/release")
+            end
+          end
+        end
+
+        def initialize(functions)
+          @functions = functions.freeze
+        end
+
+        def compress(codec, data, level)
+          out_len = [0].pack("Q")
+          ptr = call(:compress, codec, data, data.bytesize, level, out_len)
+          take(ptr, out_len.unpack1("Q"))
+        end
+
+        def decompress(codec, data, expected_len)
+          out_len = [0].pack("Q")
+          ptr = call(:decompress, codec, data, data.bytesize, expected_len, out_len)
+          take(ptr, out_len.unpack1("Q"))
+        end
+
+        def last_error
+          ptr = @functions[:last_error].call
+          ptr.null? ? "unknown error" : ptr.to_s
+        end
+
+        private
+
+        def call(name, codec, input, input_len, a, out_len)
+          ptr = @functions[name].call(codec, input, input_len, a, out_len)
+          raise Error, last_error if ptr.null?
+
+          ptr
+        end
+
+        # Copy out + free in one breath: the buffer belongs to the
+        # Rust allocator and must never outlive ozip_free.
+        def take(ptr, len)
+          string = ptr.to_s(len)
+          @functions[:free].call(ptr, len)
+          string
+        end
+      end
+
+      # Raised when a codec call fails; #message carries the Rust
+      # side's last_error text.
+      class Error < StandardError; end
+    end
+  end
+end

data/lib/omnizip/implementations/rust.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Implementations
+    # Rust-accelerated codec backend (TODO.ref-parity/62).
+    #
+    # Loads the omnizip-ffi cdylib through stdlib Fiddle — no
+    # compiled Ruby extension, no rake-compiler. When the library
+    # is unavailable the gem transparently stays on the pure-Ruby
+    # path; nothing about the fallback is an error.
+    #
+    # Library resolution order:
+    #   1. ENV['OMNIZIP_FFI_DYLIB'] (explicit path)
+    #   2. ../target/release/ inside a sibling omnizip-rs checkout
+    #
+    # Tier policy (see Omnizip::Backends): decode is always
+    # output-invariant, so `auto` uses Rust for decode whenever
+    # available; encode uses Rust only when byte-identity with the
+    # pure-Ruby encoder has been verified for that codec
+    # (RUST_ENCODE_IDENTICAL), keeping content addressing stable.
+    module Rust
+      autoload :Library, "omnizip/implementations/rust/library"
+    end
+  end
+end

data/lib/omnizip/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Omnizip
-  VERSION = "0.3.58"
+  VERSION = "0.4.1"
 end

data/lib/omnizip.rb

@@ -18,6 +18,7 @@
 
 module Omnizip
   autoload :VERSION, "omnizip/version"
+  autoload :Backends, "omnizip/backends"
 
   autoload :Error, "omnizip/error"
   autoload :CompressionError, "omnizip/error"
@@ -120,6 +121,7 @@ module Omnizip
   module Implementations
     autoload :SevenZip, "omnizip/implementations/seven_zip"
     autoload :XZUtils, "omnizip/implementations/xz_utils"
+    autoload :Rust, "omnizip/implementations/rust"
 
     module SevenZip
       module LZMA

data/lib/tasks/rust_backend.rake

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# One command to build the Rust-accelerated backend's cdylib into
+# the gem's vendor dir (TODO.ref-parity/62 — the distribution
+# story). Requires git + a Rust toolchain; the gem stays fully
+# functional (pure Ruby) without ever running this.
+#
+#   bundle exec rake rust:build
+#   bundle exec rake rust:build REPO=/path/to/omnizip-rs
+desc "Build the omnizip-ffi cdylib into vendor/ for the Rust tier"
+task "rust:build" do
+  repo = ENV.fetch("REPO", nil)
+  gem_root = File.expand_path("../..", __dir__)
+  build = File.join(gem_root, "build", "rust")
+
+  require "fileutils"
+  FileUtils.mkdir_p(build)
+  src = if repo
+          File.expand_path(repo)
+        else
+          target = File.join(build, "omnizip-rs")
+          if File.directory?(File.join(target, ".git"))
+            system("git", "-C", target, "fetch", "--quiet", "origin") or abort "git fetch failed in #{target}"
+            system("git", "-C", target, "reset", "--quiet", "--hard", "origin/main") or abort "git reset failed"
+          else
+            system("git", "clone", "--quiet", "--depth", "1",
+                   "https://github.com/omnizip/omnizip-rs.git", target) or abort "clone failed"
+          end
+          target
+        end
+
+  abort "#{src} has no omnizip-ffi crate" unless File.directory?(File.join(src, "omnizip-ffi"))
+
+  system("cargo", "build", "--release", "-p", "omnizip-ffi", chdir: src) or abort "cargo build failed"
+
+  vendor = File.join(gem_root, "vendor")
+  FileUtils.mkdir_p(vendor)
+  %w[dylib so dll].each do |ext|
+    candidate = File.join(src, "target", "release", "libomnizip_ffi.#{ext}")
+    next unless File.file?(candidate)
+
+    FileUtils.cp(candidate, vendor)
+    puts "vendored: #{File.join(vendor, "libomnizip_ffi.#{ext}")}"
+    puts "tier: OMNIZIP_BACKEND=auto now uses the Rust decoders"
+    exit 0
+  end
+  abort "cdylib not found under #{src}/target/release"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: omnizip
 version: !ruby/object:Gem::Version
-  version: 0.3.58
+  version: 0.4.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-03 00:00:00.000000000 Z
+date: 2026-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -211,6 +211,7 @@ files:
 - docs/reference/index.adoc
 - docs/resources/faq.adoc
 - docs/resources/quick-reference.adoc
+- docs/rust-backend.md
 - docs/troubleshooting/index.adoc
 - docs/xar_format.md
 - examples/api_comparison.rb
@@ -334,6 +335,7 @@ files:
 - lib/omnizip/archive_handlers/tar_handler.rb
 - lib/omnizip/archive_handlers/xar_handler.rb
 - lib/omnizip/archive_handlers/zip_handler.rb
+- lib/omnizip/backends.rb
 - lib/omnizip/buffer.rb
 - lib/omnizip/buffer/memory_archive.rb
 - lib/omnizip/buffer/memory_extractor.rb
@@ -598,6 +600,8 @@ files:
 - lib/omnizip/implementations/base/lzma_decoder_base.rb
 - lib/omnizip/implementations/base/lzma_encoder_base.rb
 - lib/omnizip/implementations/base/state_machine_base.rb
+- lib/omnizip/implementations/rust.rb
+- lib/omnizip/implementations/rust/library.rb
 - lib/omnizip/implementations/seven_zip.rb
 - lib/omnizip/implementations/seven_zip/lzma.rb
 - lib/omnizip/implementations/seven_zip/lzma/decoder.rb
@@ -719,6 +723,7 @@ files:
 - lib/omnizip/zip/input_stream.rb
 - lib/omnizip/zip/output_stream.rb
 - lib/tasks/msi_test_files.rake
+- lib/tasks/rust_backend.rake
 - readme-docs/advanced-features.adoc
 - readme-docs/api-usage.adoc
 - readme-docs/architecture.adoc