omnizip 0.4.7-arm-linux

data/lib/omnizip/algorithms/deflate64/encoder.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Omnizip
+  module Algorithms
+    class Deflate64
+      # Deflate64 encoder
+      class Encoder
+        include Constants
+
+        attr_reader :window_size
+
+        def initialize(output_stream, options = {})
+          @output_stream = output_stream
+          @window_size = options[:window_size] || DICTIONARY_SIZE
+          @compression_level = options[:level] || 6
+          @lz77_encoder = LZ77Encoder.new(@window_size)
+          @huffman = HuffmanCoder.new
+        end
+
+        # Compress input stream to output stream
+        #
+        # @param input_stream [IO] Input data stream
+        def compress(input_stream)
+          data = input_stream.read
+
+          # Step 1: LZ77 compression with 64KB window
+          tokens = @lz77_encoder.find_matches(data)
+
+          # Step 2: Huffman coding
+          compressed = @huffman.encode(tokens)
+
+          # Step 3: Serialize trees and write to output
+          output = serialize_with_trees(
+            compressed,
+            @huffman.literal_tree,
+            @huffman.distance_tree,
+          )
+
+          @output_stream.write(output)
+        end
+
+        private
+
+        # Serialize compressed data with Huffman trees
+        #
+        # @param compressed [String] Compressed data
+        # @param literal_tree [Hash] Literal Huffman tree
+        # @param distance_tree [Hash] Distance Huffman tree
+        # @return [String] Serialized output
+        def serialize_with_trees(compressed, literal_tree, distance_tree)
+          literal_json = literal_tree.to_json
+          distance_json = distance_tree.to_json
+
+          # Pack: literal_size (4 bytes), distance_size (4 bytes),
+          #       literal_tree, distance_tree, compressed_data
+          [
+            literal_json.bytesize,
+            distance_json.bytesize,
+            literal_json,
+            distance_json,
+            compressed,
+          ].pack("NNA#{literal_json.bytesize}A#{distance_json.bytesize}A*")
+        end
+
+        # Encode data block
+        #
+        # @param data [String] Input data
+        # @return [String] Compressed data
+        def encode_block(data)
+          # Find LZ77 matches
+          tokens = @lz77_encoder.find_matches(data)
+
+          # Huffman encode
+          @huffman.encode(tokens)
+        end
+
+        # Encode stream in chunks
+        #
+        # @param input_stream [IO] Input stream
+        # @param chunk_size [Integer] Size of chunks to process
+        def encode_stream(input_stream, chunk_size = 65_536)
+          until input_stream.eof?
+            chunk = input_stream.read(chunk_size)
+            break if chunk.nil? || chunk.empty?
+
+            compressed = encode_block(chunk)
+            @output_stream.write(compressed)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/deflate64/huffman_coder.rb

@@ -0,0 +1,352 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Algorithms
+    class Deflate64
+      # Huffman coding for Deflate64
+      class HuffmanCoder
+        include Constants
+
+        # Length code mapping
+        LENGTH_CODES_MAP = {
+          3 => 257, 4 => 258, 5 => 259, 6 => 260, 7 => 261,
+          8 => 262, 9 => 263, 10 => 264, 11 => 265, 12 => 266,
+          13 => 267, 14 => 268, 15 => 269, 16 => 270, 17 => 271,
+          18 => 272, 19 => 273, 20 => 274, 21 => 275, 22 => 276,
+          23 => 277, 24 => 278, 25 => 279, 26 => 280, 27 => 281,
+          28 => 282, 29 => 283, 30 => 284, 31 => 285
+        }.freeze
+
+        # Distance code mapping
+        DISTANCE_CODES_MAP = (0..29).to_a.freeze
+
+        attr_reader :literal_tree, :distance_tree
+
+        def initialize
+          @literal_tree = nil
+          @distance_tree = nil
+        end
+
+        # Encode tokens using Huffman coding
+        #
+        # @param tokens [Array<Hash>] LZ77 tokens
+        # @return [String] Encoded bitstream
+        def encode(tokens)
+          # Build frequency tables
+          literal_freqs = build_literal_frequencies(tokens)
+          distance_freqs = build_distance_frequencies(tokens)
+
+          # Build Huffman trees
+          @literal_tree = build_tree(literal_freqs)
+          @distance_tree = build_tree(distance_freqs)
+
+          # Encode tokens
+          encode_tokens(tokens)
+        end
+
+        # Decode bitstream using Huffman coding
+        #
+        # @param bitstream [String] Encoded data
+        # @param literal_tree [Hash] Literal Huffman tree
+        # @param distance_tree [Hash] Distance Huffman tree
+        # @return [Array<Hash>] Decoded tokens
+        def decode(bitstream, literal_tree, distance_tree)
+          @literal_tree = literal_tree
+          @distance_tree = distance_tree
+
+          decode_tokens(bitstream)
+        end
+
+        private
+
+        # Build frequency table for literals and lengths
+        #
+        # @param tokens [Array<Hash>] LZ77 tokens
+        # @return [Hash] Frequency table
+        def build_literal_frequencies(tokens)
+          freqs = Hash.new(0)
+
+          tokens.each do |token|
+            if token[:type] == :literal
+              freqs[token[:value]] += 1
+            else
+              length_code = length_to_code(token[:length])
+              freqs[length_code] += 1
+            end
+          end
+
+          # Add end of block marker
+          freqs[END_OF_BLOCK] = 1
+
+          freqs
+        end
+
+        # Build frequency table for distances
+        #
+        # @param tokens [Array<Hash>] LZ77 tokens
+        # @return [Hash] Frequency table
+        def build_distance_frequencies(tokens)
+          freqs = Hash.new(0)
+
+          tokens.each do |token|
+            next unless token[:type] == :match
+
+            distance_code = distance_to_code(token[:distance])
+            freqs[distance_code] += 1
+          end
+
+          freqs
+        end
+
+        # Build Huffman tree from frequencies
+        #
+        # @param frequencies [Hash] Symbol frequencies
+        # @return [Hash] Huffman code table
+        def build_tree(frequencies)
+          return {} if frequencies.empty?
+
+          # Build priority queue of nodes
+          nodes = frequencies.map do |symbol, freq|
+            { symbol: symbol, freq: freq, left: nil, right: nil }
+          end
+
+          # Build tree using priority queue
+          while nodes.size > 1
+            nodes.sort_by! { |n| n[:freq] }
+            left = nodes.shift
+            right = nodes.shift
+
+            parent = {
+              symbol: nil,
+              freq: left[:freq] + right[:freq],
+              left: left,
+              right: right,
+            }
+
+            nodes << parent
+          end
+
+          # Generate codes from tree
+          generate_codes(nodes.first)
+        end
+
+        # Generate Huffman codes from tree
+        #
+        # @param node [Hash] Tree node
+        # @param code [String] Current code
+        # @param codes [Hash] Code table
+        # @return [Hash] Complete code table
+        def generate_codes(node, code = "", codes = {})
+          return codes if node.nil?
+
+          if node[:symbol]
+            codes[node[:symbol]] = code
+          else
+            generate_codes(node[:left], "#{code}0", codes)
+            generate_codes(node[:right], "#{code}1", codes)
+          end
+
+          codes
+        end
+
+        # Encode tokens to bitstream
+        #
+        # @param tokens [Array<Hash>] LZ77 tokens
+        # @return [String] Encoded bitstream
+        def encode_tokens(tokens)
+          bits = ""
+
+          tokens.each do |token|
+            if token[:type] == :literal
+              bits += @literal_tree[token[:value]]
+            else
+              length_code = length_to_code(token[:length])
+              bits += @literal_tree[length_code]
+
+              distance_code = distance_to_code(token[:distance])
+              bits += @distance_tree[distance_code]
+            end
+          end
+
+          # Add end of block marker
+          bits += @literal_tree[END_OF_BLOCK]
+
+          # Convert bits to bytes
+          bits_to_bytes(bits)
+        end
+
+        # Decode tokens from bitstream
+        #
+        # @param bitstream [String] Encoded data
+        # @return [Array<Hash>] Decoded tokens
+        def decode_tokens(bitstream)
+          tokens = []
+          bits = bytes_to_bits(bitstream)
+          pos = 0
+
+          while pos < bits.length
+            symbol, length = decode_symbol(bits, pos, @literal_tree)
+
+            # Check for decoding failure
+            if symbol.nil?
+              raise Omnizip::DecompressionError,
+                    "Failed to decode symbol at bit position #{pos}"
+            end
+
+            pos += length
+
+            break if symbol == END_OF_BLOCK
+
+            if symbol < 256
+              tokens << { type: :literal, value: symbol }
+            else
+              match_length = code_to_length(symbol)
+
+              dist_symbol, dist_length =
+                decode_symbol(bits, pos, @distance_tree)
+
+              # Check for distance decoding failure
+              if dist_symbol.nil?
+                raise Omnizip::DecompressionError,
+                      "Failed to decode distance at bit position #{pos}"
+              end
+
+              pos += dist_length
+
+              distance = code_to_distance(dist_symbol)
+
+              tokens << {
+                type: :match,
+                length: match_length,
+                distance: distance,
+              }
+            end
+          end
+
+          tokens
+        end
+
+        # Decode single symbol from bitstream
+        #
+        # @param bits [String] Bit string
+        # @param pos [Integer] Current position
+        # @param tree [Hash] Huffman tree
+        # @return [Array] Symbol and bits consumed
+        def decode_symbol(bits, pos, tree)
+          code = ""
+          reverse_tree = tree.invert
+
+          while pos < bits.length
+            code += bits[pos]
+            pos += 1
+
+            if reverse_tree[code]
+              return [reverse_tree[code], code.length]
+            end
+          end
+
+          [nil, 0]
+        end
+
+        # Convert match length to Huffman code
+        def length_to_code(length)
+          LENGTH_CODES_MAP[length] || 285
+        end
+
+        # Convert Huffman code to match length
+        def code_to_length(code)
+          LENGTH_CODES_MAP.key(code) || 258
+        end
+
+        # Convert distance to Huffman code
+        # Uses DEFLATE distance code table
+        def distance_to_code(distance)
+          case distance
+          when 1..4
+            distance - 1
+          when 5..8
+            4 + ((distance - 5) / 2)
+          when 9..16
+            6 + ((distance - 9) / 4)
+          when 17..32
+            8 + ((distance - 17) / 8)
+          when 33..64
+            10 + ((distance - 33) / 16)
+          when 65..128
+            12 + ((distance - 65) / 32)
+          when 129..256
+            14 + ((distance - 129) / 64)
+          when 257..512
+            16 + ((distance - 257) / 128)
+          when 513..1024
+            18 + ((distance - 513) / 256)
+          when 1025..2048
+            20 + ((distance - 1025) / 512)
+          when 2049..4096
+            22 + ((distance - 2049) / 1024)
+          when 4097..8192
+            24 + ((distance - 4097) / 2048)
+          when 8193..16384
+            26 + ((distance - 8193) / 4096)
+          when 16385..32768
+            28 + ((distance - 16385) / 8192)
+          when 32769..65536
+            29
+          else
+            29 # Max distance code for 64KB window
+          end
+        end
+
+        # Convert Huffman code to distance
+        # Uses DEFLATE distance code table (base distances)
+        def code_to_distance(code)
+          case code
+          when 0..3
+            code + 1
+          when 4..5
+            5 + ((code - 4) * 2)
+          when 6..7
+            9 + ((code - 6) * 4)
+          when 8..9
+            17 + ((code - 8) * 8)
+          when 10..11
+            33 + ((code - 10) * 16)
+          when 12..13
+            65 + ((code - 12) * 32)
+          when 14..15
+            129 + ((code - 14) * 64)
+          when 16..17
+            257 + ((code - 16) * 128)
+          when 18..19
+            513 + ((code - 18) * 256)
+          when 20..21
+            1025 + ((code - 20) * 512)
+          when 22..23
+            2049 + ((code - 22) * 1024)
+          when 24..25
+            4097 + ((code - 24) * 2048)
+          when 26..27
+            8193 + ((code - 26) * 4096)
+          when 28..29
+            16385 + ((code - 28) * 8192)
+          else
+            1 # Default to distance 1
+          end
+        end
+
+        # Convert bit string to bytes
+        def bits_to_bytes(bits)
+          bytes = bits.scan(/.{1,8}/).map do |byte_bits|
+            byte_bits.ljust(8, "0").to_i(2)
+          end
+          bytes.pack("C*")
+        end
+
+        # Convert bytes to bit string
+        def bytes_to_bits(bytes)
+          bytes.unpack("C*").map { |b| b.to_s(2).rjust(8, "0") }.join
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/deflate64/lz77_encoder.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Algorithms
+    class Deflate64
+      # LZ77 encoder with 64KB sliding window for Deflate64
+      class LZ77Encoder
+        include Constants
+
+        attr_reader :window_size
+
+        def initialize(window_size = DICTIONARY_SIZE)
+          @window_size = window_size
+          @window = []
+          @hash_table = {}
+          @position = 0
+        end
+
+        # Find matches in data and return array of literals and match tokens
+        #
+        # @param data [String] Input data to compress
+        # @return [Array<Hash>] Array of match tokens
+        def find_matches(data)
+          tokens = []
+          pos = 0
+
+          while pos < data.bytesize
+            match = find_longest_match(pos, data)
+
+            if match && match[:length] >= MIN_MATCH_LENGTH
+              tokens << {
+                type: :match,
+                length: match[:length],
+                distance: match[:distance],
+              }
+              pos += match[:length]
+            else
+              tokens << {
+                type: :literal,
+                value: data.getbyte(pos),
+              }
+              pos += 1
+            end
+
+            update_window(data, pos)
+          end
+
+          tokens
+        end
+
+        private
+
+        # Find longest match for current position
+        #
+        # @param pos [Integer] Current position in data
+        # @param data [String] Input data
+        # @return [Hash, nil] Match information or nil
+        def find_longest_match(pos, data)
+          return nil if pos + MIN_MATCH_LENGTH > data.bytesize
+
+          best_match = nil
+          best_length = MIN_MATCH_LENGTH - 1
+
+          # Calculate hash for current position
+          hash = calculate_hash(data, pos)
+          candidates = @hash_table[hash] || []
+
+          # Search through candidate matches
+          candidates.reverse.take(MAX_CHAIN_LENGTH).each do |candidate_pos|
+            distance = pos - candidate_pos
+            break if distance > MAX_DISTANCE
+
+            # Find match length
+            length = match_length(data, pos, candidate_pos)
+
+            if length > best_length
+              best_length = length
+              best_match = {
+                length: length,
+                distance: distance,
+              }
+
+              break if length >= NICE_MATCH
+            end
+          end
+
+          # Add current position to hash table
+          @hash_table[hash] ||= []
+          @hash_table[hash] << pos
+
+          best_match
+        end
+
+        # Calculate match length between two positions
+        #
+        # @param data [String] Input data
+        # @param pos1 [Integer] First position
+        # @param pos2 [Integer] Second position
+        # @return [Integer] Match length
+        def match_length(data, pos1, pos2)
+          max_length = [MAX_MATCH_LENGTH, data.bytesize - pos1].min
+          length = 0
+
+          while length < max_length &&
+              data.getbyte(pos1 + length) == data.getbyte(pos2 + length)
+            length += 1
+          end
+
+          length
+        end
+
+        # Calculate hash value for position
+        #
+        # @param data [String] Input data
+        # @param pos [Integer] Position to hash
+        # @return [Integer] Hash value
+        def calculate_hash(data, pos)
+          return 0 if pos + MIN_MATCH_LENGTH > data.bytesize
+
+          hash = 0
+          MIN_MATCH_LENGTH.times do |i|
+            hash = ((hash << HASH_SHIFT) ^
+              data.getbyte(pos + i)) & (HASH_SIZE - 1)
+          end
+          hash
+        end
+
+        # Update sliding window
+        #
+        # @param data [String] Input data
+        # @param pos [Integer] Current position
+        def update_window(data, pos)
+          @window << data.getbyte(pos - 1) if pos.positive?
+          @window.shift if @window.size > @window_size
+          @position = pos
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/deflate64.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module Omnizip
+  module Algorithms
+    # Deflate64 (Enhanced Deflate) compression algorithm
+    #
+    # Extends standard Deflate with:
+    # - 64KB sliding window (vs 32KB)
+    # - Better compression for large files
+    # - ZIP compression method 9
+    #
+    # NOTE: This is a simplified implementation that uses standard
+    # Deflate internally, as true Deflate64 requires complex
+    # bit-level manipulation that is better handled by libraries
+    # specifically designed for it.
+    class Deflate64 < Algorithm
+      # Nested classes - autoloaded
+      autoload :Constants, "omnizip/algorithms/deflate64/constants"
+      autoload :Decoder, "omnizip/algorithms/deflate64/decoder"
+      autoload :Encoder, "omnizip/algorithms/deflate64/encoder"
+      autoload :HuffmanCoder, "omnizip/algorithms/deflate64/huffman_coder"
+      autoload :Lz77Encoder, "omnizip/algorithms/deflate64/lz77_encoder"
+
+      # Constants
+      DICTIONARY_SIZE = 65_536 # 64KB window
+
+      # Algorithm metadata
+      def self.metadata
+        Models::AlgorithmMetadata.new.tap do |meta|
+          meta.name = "deflate64"
+          meta.description = "Enhanced Deflate with 64KB window"
+          meta.version = "1.0.0"
+        end
+      end
+
+      # Compress input stream to output stream
+      #
+      # @param input [IO] Input stream
+      # @param output [IO] Output stream
+      # @param options [Hash] Compression options
+      # @option options [Integer] :level Compression level (1-9)
+      def compress(input, output, options = {})
+        raw_level = options[:level] || Zlib::DEFAULT_COMPRESSION
+        data = input.read
+        return if data.nil? || data.empty?
+
+        # Zlib::DEFAULT_COMPRESSION (-1) maps to the FFI's 6.
+        level = raw_level.negative? ? 6 : raw_level.clamp(0, 9)
+        compressed = Backends.compress("zlib", data, level) do
+          deflater = Zlib::Deflate.new(
+            raw_level,
+            Zlib::MAX_WBITS,
+            Zlib::MAX_MEM_LEVEL,
+          )
+          result = deflater.deflate(data, Zlib::FINISH)
+          deflater.close
+          result
+        end
+        output.write(compressed)
+      end
+
+      # Decompress input stream to output stream
+      #
+      # @param input [IO] Input stream
+      # @param output [IO] Output stream
+      # @param options [Hash] Decompression options
+      def decompress(input, output, _options = {})
+        compressed = input.read
+        return if compressed.nil? || compressed.empty?
+
+        # Set output to binary mode if it's a StringIO
+        output.set_encoding(Encoding::BINARY)
+        output.binmode
+
+        # The streams this algorithm produces/consumes are zlib
+        # containers (RFC 1950); the Rust "zlib" name speaks exactly
+        # that, so the tier swap is byte-transparent.
+        decompressed = Backends.decompress("zlib", compressed, Backends::UNKNOWN_LENGTH) do
+          inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
+          result = inflater.inflate(compressed)
+          inflater.close
+          result
+        end
+
+        # Force binary encoding to match original data
+        decompressed.force_encoding(Encoding::BINARY)
+
+        output.write(decompressed)
+      end
+
+      # Check if streaming is supported
+      #
+      # @return [Boolean] Always true for Deflate64
+      def self.streaming_supported?
+        true
+      end
+
+      # Get dictionary size
+      #
+      # @return [Integer] 64KB
+      def self.dictionary_size
+        DICTIONARY_SIZE
+      end
+
+      # Get compression method ID for ZIP format
+      #
+      # @return [Integer] Method 9
+      def self.compression_method
+        9
+      end
+    end
+  end
+end
+
+# Register algorithm
+Omnizip::AlgorithmRegistry.register(:deflate64, Omnizip::Algorithms::Deflate64)