omnizip 0.3.20 → 0.3.22

data/lib/omnizip/algorithms/zstandard/dictionary.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Zstandard dictionary (port of the omnizip-rs dict.rs).
+      #
+      # A dictionary lets the encoder preload a reference-content
+      # window so small inputs compress dramatically better: the
+      # dictionary content is used as a match-finder prefix, and the
+      # frame header carries the dictionary's ID.
+      #
+      # Wire format (simplified form, as in the Rust reference):
+      # magic (4) + dictionary ID (4, LE) + raw content.
+      class Dictionary
+        DICT_MAGIC = 0xEC30A437
+
+        attr_reader :id, :content
+
+        # @param id [Integer] dictionary ID carried in frame headers
+        # @param content [String] corpus bytes used as the prefix
+        def initialize(id, content)
+          @id = id
+          @content = content.dup.force_encoding(Encoding::BINARY)
+        end
+
+        def self.from_raw(id, content)
+          new(id, content)
+        end
+
+        def self.deserialize(data)
+          if data.bytesize < 8
+            raise Omnizip::DecompressionError,
+                  "dictionary too short for magic + id"
+          end
+          if data.byteslice(0, 4).unpack1("V") != DICT_MAGIC
+            raise Omnizip::DecompressionError, "bad dictionary magic"
+          end
+
+          new(data.byteslice(4, 4).unpack1("V"), data.byteslice(8..))
+        end
+
+        def serialize
+          [DICT_MAGIC].pack("V") + [@id].pack("V") + @content
+        end
+
+        def ==(other)
+          other.is_a?(Dictionary) && id == other.id && content == other.content
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/encoder.rb

@@ -58,42 +58,83 @@ module Omnizip
           write_frame(data)
         end
 
+        # Encode a data stream primed with a dictionary prefix (port
+        # of the Rust encode_frame_with_dict): the dictionary content
+        # is prepended as virtual history, the match finder is seeded
+        # with its positions, and only the plaintext is emitted.
+        # Frame_Content_Size and the checksum cover the plaintext
+        # only; the frame header carries the dictionary's ID and
+        # requires the dict-aware decoder.
+        #
+        # @param data [String]
+        # @param dict [Dictionary]
+        def encode_stream_with_dict(data, dict)
+          data = data.dup.force_encoding(Encoding::BINARY)
+          virtual = dict.content + data
+          prefix_len = dict.content.bytesize
+
+          write_u32le(MAGIC_NUMBER)
+          write_frame_header(data, dict.id)
+          write_blocks(virtual, prefix_len)
+          write_u32le(XXHash64.frame_checksum(data)) if @checksum
+        end
+
         private
 
-        def write_frame(data)
+        def write_frame(data, dict = nil)
           write_u32le(MAGIC_NUMBER)
-          write_frame_header(data)
+          write_frame_header(data, dict && dict.id)
           write_blocks(data)
           write_u32le(XXHash64.frame_checksum(data)) if @checksum
         end
 
-        # Single-segment frame header: no window descriptor, no
-        # dictionary; Frame_Content_Size in 1, 2, 4 or 8 bytes.
-        def write_frame_header(data)
+        # Single-segment frame header: no window descriptor;
+        # optional Dictionary_ID, then Frame_Content_Size in 1, 2, 4
+        # or 8 bytes.
+        def write_frame_header(data, dict_id = nil)
+          did_flag, did_bytes = dict_id_encoding(dict_id)
+
           size = data.bytesize
           if size < 256
-            @output_stream.putc(0x20) # single segment, FCS 1 byte
+            @output_stream.putc(0x20 | did_flag) # single segment, FCS 1 byte
+            write_dict_id_bytes(did_bytes)
             @output_stream.putc(size)
           elsif size <= 65_791
-            @output_stream.putc(0x60) # single segment, FCS 2 bytes (+256)
+            @output_stream.putc(0x60 | did_flag) # single segment, FCS 2 bytes
+            write_dict_id_bytes(did_bytes)
             write_u16le(size - 256)
           elsif size <= 0xFFFFFFFF
-            @output_stream.putc(0xA0) # single segment, FCS 4 bytes
+            @output_stream.putc(0xA0 | did_flag) # single segment, FCS 4 bytes
+            write_dict_id_bytes(did_bytes)
             write_u32le(size)
           else
-            @output_stream.putc(0xE0) # single segment, FCS 8 bytes
+            @output_stream.putc(0xE0 | did_flag) # single segment, FCS 8 bytes
+            write_dict_id_bytes(did_bytes)
             write_u64le(size)
           end
         end
 
+        def dict_id_encoding(dict_id)
+          return [0, "".b] if dict_id.nil? || dict_id.zero?
+          return [1, [dict_id].pack("C")] if dict_id <= 0xFF
+          return [2, [dict_id].pack("v")] if dict_id <= 0xFFFF
+
+          [3, [dict_id].pack("V")]
+        end
+
+        def write_dict_id_bytes(bytes)
+          @output_stream.write(bytes)
+        end
+
         # rubocop:disable Metrics/MethodLength
         # rubocop:disable-next Metrics/AbcSize
-        def write_blocks(data)
-          return write_empty_last_block if data.empty?
+        def write_blocks(data, start = 0)
+          return write_empty_last_block if data.bytesize <= start
 
           params = MatchFinder.params_for_level(@level, data.bytesize)
           ms = MatchFinder::MatchState.new(params[:hash_log])
           ms.enable_chain(params[:chain]) if params[:chain].positive?
+          ms.seed_prefix(data, start) if start.positive?
 
           # LDM (long-distance matching) at high levels on multi-block
           # inputs: a sparse table over the whole frame finds matches
@@ -115,32 +156,55 @@ module Omnizip
           # untouched.
           reps = [1, 4, 8]
 
-          offset = 0
+          offset = start
           while offset < data.bytesize
             block_end = [offset + BLOCK_MAX_SIZE, data.bytesize].min
             is_last = block_end == data.bytesize
             chunk = data.byteslice(offset, block_end - offset)
 
-            if chunk.bytesize >= 4 && rle_chunk?(chunk)
-              write_block_header(is_last ? 1 : 0, BLOCK_TYPE_RLE,
-                                 chunk.bytesize)
-              @output_stream.putc(chunk.getbyte(0))
-            else
-              seq_store = MatchFinder::SeqStore.new(reps.dup)
-              MatchFinder.compress_range(data, offset, block_end, seq_store,
-                                         ms, params[:min_match],
-                                         params[:lazy], ldm, max_distance)
-              content, new_reps = try_compressed(seq_store, reps)
-              if content.nil?
-                write_block_header(is_last ? 1 : 0, BLOCK_TYPE_RAW,
-                                   chunk.bytesize)
-                @output_stream.write(chunk)
+            # Adaptive splitting (port of the Rust reference): a
+            # heterogeneous chunk — first and second halves' byte
+            # distributions diverge — is split into 16 KiB sub-blocks
+            # so each Huffman/FSE table fits its region; homogeneous
+            # chunks keep the single block and its minimal header
+            # overhead.
+            step = if chunk.bytesize >= 32 * 1024 &&
+                halves_diverge(chunk)
+                     16 * 1024
+                   else
+                     chunk.bytesize
+                   end
+
+            sub = offset
+            while sub < block_end
+              sub_end = [sub + step, block_end].min
+              sub_last = is_last && sub_end == block_end
+              sub_chunk = data.byteslice(sub, sub_end - sub)
+
+              if sub_chunk.bytesize >= 4 && rle_chunk?(sub_chunk)
+                write_block_header(sub_last ? 1 : 0, BLOCK_TYPE_RLE,
+                                   sub_chunk.bytesize)
+                @output_stream.putc(sub_chunk.getbyte(0))
               else
-                write_block_header(is_last ? 1 : 0, BLOCK_TYPE_COMPRESSED,
-                                   content.bytesize)
-                @output_stream.write(content)
-                reps = new_reps
+                seq_store = MatchFinder::SeqStore.new(reps.dup)
+                MatchFinder.compress_range(data, sub, sub_end, seq_store,
+                                           ms, params[:min_match],
+                                           params[:lazy], ldm, max_distance)
+                content, new_reps = try_compressed(seq_store, reps)
+                if content.nil?
+                  write_block_header(sub_last ? 1 : 0, BLOCK_TYPE_RAW,
+                                     sub_chunk.bytesize)
+                  @output_stream.write(sub_chunk)
+                else
+                  write_block_header(sub_last ? 1 : 0,
+                                     BLOCK_TYPE_COMPRESSED,
+                                     content.bytesize)
+                  @output_stream.write(content)
+                  reps = new_reps
+                end
               end
+
+              sub = sub_end
             end
 
             offset = block_end
@@ -152,6 +216,22 @@ module Omnizip
           write_block_header(1, BLOCK_TYPE_RAW, 0)
         end
 
+        # Whether the byte distributions of a chunk's halves diverge
+        # (total-variation sum above 0.5 = "different content
+        # regimes", per the Rust reference).
+        def halves_diverge(chunk)
+          mid = chunk.bytesize / 2
+          ha = Array.new(256, 0)
+          hb = Array.new(256, 0)
+          chunk.byteslice(0, mid).each_byte { |b| ha[b] += 1 }
+          chunk.byteslice(mid, chunk.bytesize - mid).each_byte { |b| hb[b] += 1 }
+          la = mid.to_f
+          lb = (chunk.bytesize - mid).to_f
+          tvd = 0.0
+          256.times { |i| tvd += ((ha[i] / la) - (hb[i] / lb)).abs }
+          tvd > 0.5
+        end
+
         # Build a Compressed_Block content: literals section plus the
         # sequences section. Returns [content, wire_reps], or [nil,
         # nil] when the result is not smaller than the raw chunk.

data/lib/omnizip/algorithms/zstandard/match_finder.rb

@@ -88,6 +88,18 @@ module Omnizip
           def disable_chain
             @max_chain = 0
           end
+
+          # Seed the hash table with a prefix's positions (dictionary
+          # content prepended to the plaintext): the most recent
+          # position per hash wins, matching insert order.
+          def seed_prefix(src, prefix_len)
+            return if prefix_len < MIN_MATCH
+
+            limit = prefix_len - MIN_MATCH + 1
+            (0...limit).each do |pos|
+              @hash_table[MatchFinder.hash4(src, pos, @hash_log)] = pos
+            end
+          end
         end
 
         module_function
@@ -178,15 +190,18 @@ module Omnizip
           limit = block_end - mm
 
           while ip < limit
-            match = if ldm
+            match = if ldm || lazy.positive?
+                      # Lazy levels share the LDM loop's rep0 fast-path
+                      # (with backward extension): without it, levels
+                      # 6+ measured worse than the greedy default level
+                      # because every rep-offset match paid full offset
+                      # coding.
                       find_match_ldm(src, ip, ms, mm, limit, anchor,
                                      seq_store.rep_offsets[0], ldm,
                                      max_distance)
-                    elsif lazy.zero?
+                    else
                       find_greedy_match(src, ip, ms, mm, limit, anchor,
                                         seq_store.rep_offsets[0])
-                    else
-                      find_best_match(src, ip, ms, mm, limit)
                     end
             if match
               dist, len = match
@@ -247,7 +262,8 @@ module Omnizip
                                       ldm, max_distance)
           return nil unless dist
 
-          [dist, len, ip]
+          back = backward_extension(src, ip, anchor, ip - dist)
+          [dist, len + back, ip - back]
         end
 
         # Greedy-path match search (port of the Rust

data/lib/omnizip/algorithms/zstandard.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "stringio"
+
 # Copyright (C) 2025 Ribose Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
@@ -58,6 +60,7 @@ module Omnizip
                "omnizip/algorithms/zstandard/sequences_encoder"
       autoload :MatchFinder, "omnizip/algorithms/zstandard/match_finder"
       autoload :LdmHashTable, "omnizip/algorithms/zstandard/ldm"
+      autoload :Dictionary, "omnizip/algorithms/zstandard/dictionary"
       autoload :XXHash64, "omnizip/algorithms/zstandard/xxhash"
 
       # Frame and FSE modules
@@ -101,6 +104,54 @@ module Omnizip
         output_stream.write(decompressed)
       end
 
+      # Compress data primed with a dictionary prefix: the dictionary
+      # content acts as shared history the match finder can reference,
+      # dramatically improving ratios on small inputs. The resulting
+      # frame carries the dictionary's ID and requires
+      # decompress_with_dict to decode.
+      #
+      # @param input_stream [IO] plaintext
+      # @param output_stream [IO] compressed frame
+      # @param dict [Dictionary]
+      # @param options [Hash] :level, :checksum
+      # @return [void]
+      def compress_with_dict(input_stream, output_stream, dict, options = {})
+        encoder = Encoder.new(output_stream, build_encoder_options(options))
+        encoder.encode_stream_with_dict(input_stream.read, dict)
+      end
+
+      # Decompress a frame produced by compress_with_dict. The frame's
+      # dictionary ID is verified against `dict` before decoding.
+      #
+      # @param input_stream [IO] compressed frame
+      # @param output_stream [IO] plaintext
+      # @param dict [Dictionary]
+      # @return [void]
+      def decompress_with_dict(input_stream, output_stream, dict)
+        output_stream.set_encoding(Encoding::BINARY)
+        decoder = Decoder.new(input_stream)
+        output_stream.write(decoder.decode_stream_with_dict(dict))
+      end
+
+      # Class-level convenience mirrors of the dictionary API.
+      #
+      # @return [String] compressed frame / decompressed plaintext
+      def self.compress_with_dict(data, dict, **options)
+        output = StringIO.new
+        output.set_encoding(Encoding::BINARY)
+        new(options).compress_with_dict(
+          StringIO.new(data.to_s.b), output, dict, options
+        )
+        output.string
+      end
+
+      def self.decompress_with_dict(data, dict)
+        output = StringIO.new
+        output.set_encoding(Encoding::BINARY)
+        new.decompress_with_dict(StringIO.new(data.to_s.b), output, dict)
+        output.string
+      end
+
       private
 
       # Build encoder options from compression options
@@ -116,6 +167,7 @@ module Omnizip
         level = case options
                 when nil then nil
                 when Hash then options[:level]
+                # allowed: options is a public parameter; any object with #level is honored
                 else options.level if options.respond_to?(:level)
                 end
 

data/lib/omnizip/chunked/writer.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "fileutils"
+require "stringio"
+
 module Omnizip
   module Chunked
     # Write large files incrementally in chunks

data/lib/omnizip/commands/archive_extract_command.rb

@@ -16,6 +16,8 @@
 # See the COPYING file for the complete text of the license.
 #
 
+require "fileutils"
+
 module Omnizip
   module Commands
     # Command to extract .7z archives to a directory.
@@ -222,7 +224,7 @@ module Omnizip
 
         extracted.size
       ensure
-        archive&.close if archive.respond_to?(:close)
+        archive.close if archive.is_a?(Omnizip::Zip::File)
       end
 
       def extract_gzip(archive_file, output_dir, verbose)

data/lib/omnizip/commands/archive_list_command.rb

@@ -69,7 +69,7 @@ module Omnizip
         entries = if patterns || excludes
                     filter_entries(archive, patterns, excludes)
                   else
-                    archive.respond_to?(:entries) ? archive.entries : archive.to_a
+                    archive.entries
                   end
 
         if count_only
@@ -91,6 +91,8 @@ module Omnizip
       rescue StandardError => e
         raise Omnizip::CompressionError,
               "Failed to list archive: #{e.message}"
+      ensure
+        archive.close if archive.is_a?(Omnizip::Zip::File)
       end
 
       def filter_entries(archive, patterns, excludes)
@@ -102,8 +104,7 @@ module Omnizip
         # Add exclude patterns
         excludes&.each { |pattern| filter.exclude_pattern(pattern) }
 
-        entries = archive.respond_to?(:entries) ? archive.entries : archive.to_a
-        filter.filter(entries)
+        filter.filter(archive.entries)
       end
 
       def display_simple_listing_filtered(entries)

data/lib/omnizip/commands/profile_show_command.rb

@@ -33,6 +33,8 @@ module Omnizip
         puts "Solid:       #{profile.solid}"
         puts "Description: #{profile.description}"
 
+        # ProfileRegistry accepts any CompressionProfile subclass.
+        # allowed: a registered subclass may expose base_profile
         return unless profile.respond_to?(:base_profile) && profile.base_profile
 
         puts "Based on:    #{profile.base_profile.name}"

data/lib/omnizip/convenience.rb

@@ -123,6 +123,7 @@ module Omnizip
       end
 
       handler = Omnizip::ArchiveHandler.for(format)
+      # allowed: handler is a registered duck; missing add_entry raises below
       if handler.respond_to?(:add_entry)
         handler.add_entry(archive_path, entry_name, source_path)
       else
@@ -142,6 +143,7 @@ module Omnizip
     def remove_from_archive(archive_path, entry_name, format: DEFAULT_FORMAT)
       require_archive!(archive_path)
       handler = Omnizip::ArchiveHandler.for(format)
+      # allowed: handler is a registered duck; missing remove_entry raises below
       unless handler.respond_to?(:remove_entry)
         raise Omnizip::UnsupportedFormatError,
               "Format #{format.inspect} does not support removing entries"

data/lib/omnizip/extraction/filter_chain.rb

@@ -159,10 +159,14 @@ module Omnizip
           entry
         else
           # Try common filename methods
+          # Narrowing this changes which files a filter matches.
+          # allowed: entries may be foreign objects
           if entry.respond_to?(:name)
             entry.name
+          # allowed: entries may be foreign objects
           elsif entry.respond_to?(:path)
             entry.path
+          # allowed: entries may be foreign objects
           elsif entry.respond_to?(:filename)
             entry.filename
           else

data/lib/omnizip/extraction/selective_extractor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "fileutils"
+
 module Omnizip
   module Extraction
     # Coordinates selective extraction from archives
@@ -120,8 +122,10 @@ module Omnizip
       #
       # @return [Array] All entries
       def list_all
+        # allowed: public API takes any archive object, including foreign ones
         if @archive.respond_to?(:entries)
           @archive.entries
+        # allowed: public API takes any Enumerable archive
         elsif @archive.respond_to?(:each)
           @archive.to_a
         else
@@ -150,10 +154,13 @@ module Omnizip
       # @param entry [Object] Entry to read
       # @return [String] Entry content
       def read_entry_content(entry)
+        # allowed: entries come from a caller-supplied archive
         if entry.respond_to?(:read)
           entry.read
+        # allowed: rubyzip-style entries expose get_input_stream, not read
         elsif entry.respond_to?(:get_input_stream)
           entry.get_input_stream.read
+        # allowed: some archives read entry content from the archive
         elsif @archive.respond_to?(:read)
           @archive.read(entry)
         else

data/lib/omnizip/file_type.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "marcel"
+require "stringio"
 
 module Omnizip
   # File type detection module using Marcel for MIME type detection
@@ -95,16 +96,19 @@ module Omnizip
         return nil unless io
 
         # Save current position
+        # allowed: caller-supplied IO; only saved when the stream tracks one
         original_pos = io.pos if io.respond_to?(:pos)
 
         mime_type = Marcel::MimeType.for(io, name: filename)
 
         # Restore position
+        # allowed: caller-supplied IO capability probe
         io.seek(original_pos) if original_pos && io.respond_to?(:seek)
 
         mime_type
       rescue StandardError
         # Attempt to restore position even on error
+        # allowed: caller-supplied IO capability probe
         io.seek(original_pos) if original_pos && io.respond_to?(:seek)
         nil
       end

data/lib/omnizip/formats/ole.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "fileutils"
+
 module Omnizip
   module Formats
     # OLE compound document format support

data/lib/omnizip/formats/rar/decompressor.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "English"
+require "fileutils"
 require "tmpdir"
 module Omnizip
   module Formats

data/lib/omnizip/formats/rar/license_validator.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "fileutils"
+
 module Omnizip
   module Formats
     module Rar

data/lib/omnizip/formats/rar/reader.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "fileutils"
+require "stringio"
 
 module Omnizip
   module Formats
@@ -287,8 +288,6 @@ module Omnizip
         # @param entry [Models::RarEntry] File entry
         # @return [StringIO] Compressed data stream
         def read_compressed_data(entry)
-          require "stringio"
-
           # Find the entry's data offset in the archive
           File.open(@file_path, "rb") do |io|
             # Skip signature and headers

data/lib/omnizip/formats/rpm.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "fileutils"
 require "stringio"
 require "tempfile"
 

data/lib/omnizip/formats/seven_zip/writer.rb

@@ -490,7 +490,15 @@ num_files)
 
         def compress_with_lzma2(data)
           # Use 7-Zip SDK LZMA2 encoder for 7-Zip format
+          #
+          # The encoder dictionary is sized to the data but never
+          # exceeds the size announced in the coder properties
+          # (options default 8MB): the decoder allocates its window
+          # from the announced size, so a larger encoder dictionary
+          # could emit matches reading outside the decoder window.
+          announced = @options[:dict_size] || (8 * 1024 * 1024)
           dict_size = [4096, data.bytesize].max
+          dict_size = announced if announced < dict_size
 
           encoder = Omnizip::Implementations::SevenZip::LZMA2::Encoder.new(
             dict_size: dict_size,

data/lib/omnizip/formats/tar/reader.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "fileutils"
+
 module Omnizip
   module Formats
     module Tar

data/lib/omnizip/formats/xar/reader.rb

@@ -2,6 +2,8 @@
 
 require "zlib"
 require "digest"
+require "fileutils"
+require "stringio"
 
 module Omnizip
   module Formats

data/lib/omnizip/formats/xar/writer.rb

@@ -3,6 +3,7 @@
 require "zlib"
 require "digest"
 require "fileutils"
+require "stringio"
 
 module Omnizip
   module Formats

data/lib/omnizip/formats/xz.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "stringio"
+
 module Omnizip
   module Formats
     # XZ compression format