omnizip 0.4.7-arm-linux

data/lib/omnizip/formats/rar/compression/lz77_huffman/encoder.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Formats
+    module Rar
+      module Compression
+        module LZ77Huffman
+          # RAR LZ77+Huffman encoder
+          #
+          # Implements compression using LZ77 string matching combined with
+          # Huffman coding for symbol encoding.
+          #
+          # ## Simplified Huffman Tree Format (MVP)
+          #
+          # This implementation uses a simplified tree format for portability
+          # and ease of implementation. The format differs from official RAR
+          # but maintains full compatibility between encoder and decoder.
+          #
+          # ### Format Structure:
+          # ```
+          # [16-bit num_symbols] [code_lengths...]
+          #       2 bytes         512 × 4 bits = 256 bytes
+          # ```
+          #
+          # ### Details:
+          # - **Header**: 16-bit number of symbols (always 512 for MVP)
+          #   - 0-255: Literal bytes
+          #   - 256: End-of-block marker
+          #   - 257-511: LZ77 match symbols
+          #
+          # - **Code Lengths**: 4 bits per symbol × 512 symbols = 2048 bits
+          #   - Each symbol gets a 4-bit code length (0-15)
+          #   - Length 0 means symbol not used
+          #   - Lengths build canonical Huffman tree
+          #
+          # ### Trade-offs:
+          # - **Fixed Overhead**: 258 bytes (2 + 256) per block
+          # - **Simplicity**: Easy to implement and debug
+          # - **Portability**: Pure Ruby, no external dependencies
+          # - **Compatibility**: Encoder/decoder use identical format
+          #
+          # ### Real RAR Format Differences:
+          # Real RAR uses a more complex format with:
+          # - RLE compression of code lengths
+          # - Multiple Huffman tables (MC, LD, RC, LDD)
+          # - Adaptive tree updates
+          # - More efficient length encoding
+          #
+          # The simplified format is sufficient for MVP and can be upgraded
+          # to full RAR format in future versions without breaking the API.
+          #
+          # @see Decoder for decoding implementation
+          # @see HuffmanCoder for tree building
+          # @see HuffmanBuilder for code generation
+          class Encoder
+            LITERAL_SYMBOLS = (0..255)
+            END_OF_BLOCK = 256
+            MATCH_SYMBOLS = (257..511)
+            MIN_MATCH_LENGTH = 3
+            MAX_MATCH_LENGTH = 257
+
+            attr_reader :compressed_size
+
+            def initialize(output, _options = {})
+              @output = output
+              @bit_stream = BitStream.new(output, :write)
+              @match_finder = MatchFinder.new
+              @huffman_builder = HuffmanBuilder.new
+              @compressed_size = 0
+            end
+
+            def encode(input)
+              data = input.is_a?(String) ? input : input.read
+              return 0 if data.empty?
+
+              start_pos = @output.pos
+              items = collect_items(data)
+              codes = @huffman_builder.generate_codes
+              write_huffman_tree(codes)
+
+              items.each do |item|
+                if item[:type] == :literal
+                  encode_literal(item[:value], codes)
+                else
+                  encode_match(item[:offset], item[:length], codes)
+                end
+              end
+
+              encode_symbol(END_OF_BLOCK, codes)
+              @bit_stream.flush
+              @compressed_size = @output.pos - start_pos
+            end
+
+            private
+
+            def collect_items(data)
+              items = []
+              position = 0
+
+              while position < data.size
+                match = @match_finder.find_match(data.bytes, position)
+
+                if match && match.length >= MIN_MATCH_LENGTH
+                  items << { type: :match, offset: match.offset,
+                             length: match.length }
+                  match_symbol = encode_match_symbol(match.length)
+                  @huffman_builder.add_symbol(match_symbol)
+                  position += match.length
+                else
+                  byte = data.bytes[position]
+                  items << { type: :literal, value: byte }
+                  @huffman_builder.add_symbol(byte)
+                  position += 1
+                end
+              end
+
+              @huffman_builder.add_symbol(END_OF_BLOCK)
+              items
+            end
+
+            def write_huffman_tree(codes)
+              lengths = Array.new(512, 0)
+              codes.each { |symbol, (_code, length)| lengths[symbol] = length }
+              @bit_stream.write_bits(512, 16)
+              lengths.each { |length| @bit_stream.write_bits(length, 4) }
+            end
+
+            def encode_literal(byte, codes)
+              encode_symbol(byte, codes)
+            end
+
+            def encode_match(offset, length, codes)
+              match_symbol = encode_match_symbol(length)
+              encode_symbol(match_symbol, codes)
+              @bit_stream.write_bits(offset, 16) # Changed from 8 to 16 bits for 64KB window
+            end
+
+            def encode_match_symbol(length)
+              base_symbol = length - MIN_MATCH_LENGTH + 257
+              [base_symbol, 511].min
+            end
+
+            def encode_symbol(symbol, codes)
+              code, length = codes[symbol]
+              return unless code && length
+
+              @bit_stream.write_bits(code, length)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/formats/rar/compression/lz77_huffman/huffman_builder.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Formats
+    module Rar
+      module Compression
+        module LZ77Huffman
+          # Huffman tree builder for dynamic compression
+          #
+          # Builds canonical Huffman trees from symbol frequencies.
+          # Uses priority queue (heap) algorithm to construct optimal trees.
+          #
+          # Responsibilities:
+          # - ONE responsibility: Build Huffman trees and generate codes
+          # - Collect symbol frequencies
+          # - Build optimal Huffman tree
+          # - Generate canonical Huffman codes
+          # - Calculate code lengths
+          class HuffmanBuilder
+            MAX_CODE_LENGTH = 15
+
+            # Tree node for Huffman tree construction
+            class Node
+              attr_accessor :symbol, :frequency, :left, :right
+
+              def initialize(symbol, frequency)
+                @symbol = symbol
+                @frequency = frequency
+                @left = nil
+                @right = nil
+              end
+
+              def leaf?
+                @left.nil? && @right.nil?
+              end
+            end
+
+            attr_reader :frequencies
+
+            def initialize
+              @frequencies = Hash.new(0)
+            end
+
+            # Add symbol occurrence(s)
+            #
+            # @param symbol [Integer] Symbol value
+            # @param count [Integer] Number of occurrences
+            # @return [void]
+            def add_symbol(symbol, count = 1)
+              @frequencies[symbol] += count
+            end
+
+            # Build Huffman tree from frequencies
+            #
+            # Uses priority queue algorithm to build optimal tree.
+            # Returns root node of the tree.
+            #
+            # @return [Node, nil] Root node or nil if empty
+            def build_tree
+              return nil if @frequencies.empty?
+
+              if @frequencies.size == 1
+                return Node.new(@frequencies.keys.first,
+                                @frequencies.values.first)
+              end
+
+              # Create leaf nodes
+              heap = @frequencies.map { |symbol, freq| Node.new(symbol, freq) }
+              heap.sort_by!(&:frequency)
+
+              # Build tree bottom-up
+              while heap.size > 1
+                left = heap.shift
+                right = heap.shift
+
+                parent = Node.new(nil, left.frequency + right.frequency)
+                parent.left = left
+                parent.right = right
+
+                # Insert maintaining heap property
+                insert_into_heap(heap, parent)
+              end
+
+              heap.first
+            end
+
+            # Generate canonical Huffman codes
+            #
+            # Returns hash mapping symbols to [code, length] pairs.
+            # Codes are canonical (same-length codes are sequential).
+            #
+            # @return [Hash<Integer, Array(Integer, Integer)>] symbol => [code, length]
+            def generate_codes
+              root = build_tree
+              return {} if root.nil?
+
+              # Handle single symbol case
+              if root.leaf?
+                return { root.symbol => [0, 1] }
+              end
+
+              # Calculate code lengths for each symbol
+              code_lengths = {}
+              calculate_code_lengths(root, 0, code_lengths)
+
+              # Generate canonical codes from lengths
+              generate_canonical_codes(code_lengths)
+            end
+
+            # Get code lengths only (for header transmission)
+            #
+            # @return [Hash<Integer, Integer>] symbol => length
+            def code_lengths
+              root = build_tree
+              return {} if root.nil?
+
+              if root.leaf?
+                return { root.symbol => 1 }
+              end
+
+              lengths = {}
+              calculate_code_lengths(root, 0, lengths)
+              lengths
+            end
+
+            # Reset builder
+            #
+            # @return [void]
+            def reset
+              @frequencies.clear
+            end
+
+            # Check if empty
+            #
+            # @return [Boolean]
+            def empty?
+              @frequencies.empty?
+            end
+
+            # Get number of symbols
+            #
+            # @return [Integer]
+            def symbol_count
+              @frequencies.size
+            end
+
+            private
+
+            # Insert node into heap maintaining sort order
+            #
+            # @param heap [Array<Node>] Heap array
+            # @param node [Node] Node to insert
+            # @return [void]
+            def insert_into_heap(heap, node)
+              index = heap.bsearch_index do |n|
+                n.frequency >= node.frequency
+              end || heap.size
+              heap.insert(index, node)
+            end
+
+            # Calculate code lengths via tree traversal
+            #
+            # @param node [Node] Current node
+            # @param depth [Integer] Current depth
+            # @param lengths [Hash] Output hash
+            # @return [void]
+            def calculate_code_lengths(node, depth, lengths)
+              return if node.nil?
+
+              if node.leaf?
+                lengths[node.symbol] = [depth, MAX_CODE_LENGTH].min
+              else
+                calculate_code_lengths(node.left, depth + 1, lengths)
+                calculate_code_lengths(node.right, depth + 1, lengths)
+              end
+            end
+
+            # Generate canonical codes from code lengths
+            #
+            # Canonical codes have the property that codes of the same
+            # length are sequential integers.
+            #
+            # @param code_lengths [Hash<Integer, Integer>] symbol => length
+            # @return [Hash<Integer, Array(Integer, Integer)>] symbol => [code, length]
+            def generate_canonical_codes(code_lengths)
+              return {} if code_lengths.empty?
+
+              # Count symbols at each length
+              length_counts = Array.new(MAX_CODE_LENGTH + 1, 0)
+              code_lengths.each_value { |len| length_counts[len] += 1 }
+
+              # Calculate first code for each length
+              first_codes = Array.new(MAX_CODE_LENGTH + 1, 0)
+              code = 0
+              (1..MAX_CODE_LENGTH).each do |len|
+                first_codes[len] = code
+                code = (code + length_counts[len]) << 1
+              end
+
+              # Assign codes to symbols
+              codes = {}
+              code_lengths.sort_by do |sym, len|
+                [len, sym]
+              end.each do |symbol, length|
+                code = first_codes[length]
+                first_codes[length] += 1
+                codes[symbol] = [code, length]
+              end
+
+              codes
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/formats/rar/compression/lz77_huffman/huffman_coder.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Formats
+    module Rar
+      module Compression
+        module LZ77Huffman
+          # Huffman coding for RAR LZ77+Huffman compression
+          #
+          # Implements canonical Huffman tree decoding for RAR archives.
+          # RAR uses multiple Huffman tables:
+          # - MC (Main Code): Literals and length codes
+          # - LD (Length-Distance): Distance codes
+          # - RC (Repeat Count): Run-length encoding
+          # - LDD (Low Distance): Low distance values
+          #
+          # Responsibilities:
+          # - ONE responsibility: Huffman tree operations
+          # - Build canonical Huffman trees from code lengths
+          # - Decode symbols using Huffman trees
+          # - Parse tree structure from bit stream
+          #
+          # Canonical Huffman Code Properties:
+          # - Codes of same length are sequential
+          # - Shorter codes have lower values
+          # - Deterministic tree construction from lengths
+          class HuffmanCoder
+            # Maximum code length for RAR
+            MAX_CODE_LENGTH = 15
+
+            # Initialize Huffman coder
+            def initialize
+              @decode_table = {}
+              @code_lengths = []
+            end
+
+            # Build Huffman tree from code lengths
+            #
+            # Constructs a canonical Huffman tree given the code lengths
+            # for each symbol. This is how RAR transmits Huffman tables.
+            #
+            # @param code_lengths [Array<Integer>] Code length for each symbol
+            # @return [void]
+            def build_tree(code_lengths)
+              @code_lengths = code_lengths
+              @decode_table = {}
+
+              # Count codes of each length
+              length_counts = Array.new(MAX_CODE_LENGTH + 1, 0)
+              code_lengths.each do |len|
+                length_counts[len] += 1 if len.positive?
+              end
+
+              # Calculate first code for each length
+              first_codes = Array.new(MAX_CODE_LENGTH + 1, 0)
+              code = 0
+              (1..MAX_CODE_LENGTH).each do |len|
+                first_codes[len] = code
+                code = (code + length_counts[len]) << 1
+              end
+
+              # Assign codes to symbols
+              code_lengths.each_with_index do |len, symbol|
+                next if len.zero?
+
+                code = first_codes[len]
+                first_codes[len] += 1
+
+                # Store in decode table: [code, length] => symbol
+                key = (code << 8) | len
+                @decode_table[key] = symbol
+              end
+            end
+
+            # Decode a single symbol from bit stream
+            #
+            # Reads bits one at a time until a valid Huffman code is found,
+            # then returns the corresponding symbol.
+            #
+            # @param bit_stream [BitStream] Input bit stream
+            # @return [Integer, nil] Decoded symbol or nil if end
+            def decode_symbol(bit_stream)
+              code = 0
+              length = 0
+
+              # Read bits until we find a valid code
+              (1..MAX_CODE_LENGTH).each do |len|
+                bit = bit_stream.read_bit
+                code = (code << 1) | bit
+                length = len
+
+                # Check if this code exists in decode table
+                key = (code << 8) | length
+                return @decode_table[key] if @decode_table.key?(key)
+              end
+
+              # No valid code found
+              nil
+            end
+
+            # Parse Huffman tree from RAR bit stream
+            #
+            # RAR encodes Huffman trees in a compact format:
+            # 1. Number of code lengths
+            # 2. Code lengths (potentially compressed)
+            # 3. Tree structure
+            #
+            # This is a simplified implementation for MVP.
+            #
+            # @param bit_stream [BitStream] Input bit stream
+            # @param num_symbols [Integer] Number of symbols in alphabet
+            # @return [void]
+            def parse_tree(bit_stream, num_symbols)
+              code_lengths = Array.new(num_symbols, 0)
+
+              # Read code lengths (simplified - real RAR uses RLE)
+              num_symbols.times do |i|
+                # Read length as 4-bit value
+                len = bit_stream.read_bits(4)
+                code_lengths[i] = len
+              end
+
+              build_tree(code_lengths)
+            end
+
+            # Check if tree is empty
+            #
+            # @return [Boolean] True if no codes defined
+            def empty?
+              @decode_table.empty?
+            end
+
+            # Get number of symbols in tree
+            #
+            # @return [Integer] Number of symbols
+            def symbol_count
+              @decode_table.size
+            end
+
+            # Reset the coder
+            #
+            # @return [void]
+            def reset
+              @decode_table = {}
+              @code_lengths = []
+            end
+
+            # Encode a symbol (for future encoder implementation)
+            #
+            # @param symbol [Integer] Symbol to encode
+            # @return [Array<Integer, Integer>] [code, length]
+            def encode_symbol(symbol)
+              # Find code for symbol
+              @decode_table.each do |key, sym|
+                next unless sym == symbol
+
+                code = key >> 8
+                length = key & 0xFF
+                return [code, length]
+              end
+
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/formats/rar/compression/lz77_huffman/match_finder.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Formats
+    module Rar
+      module Compression
+        module LZ77Huffman
+          # LZ77 Match Finder for RAR compression
+          class MatchFinder
+            MAX_MATCH_LENGTH = 257
+            MIN_MATCH_LENGTH = 3
+            WINDOW_SIZE = 32768
+            MAX_CHAIN_LENGTH = 1024
+
+            class Match
+              attr_reader :offset, :length
+
+              def initialize(offset, length)
+                @offset = offset
+                @length = length
+              end
+
+              def ==(other)
+                offset == other.offset && length == other.length
+              end
+            end
+
+            attr_reader :window_size, :max_match_length
+
+            def initialize(window_size = WINDOW_SIZE,
+max_match_length = MAX_MATCH_LENGTH)
+              @window_size = window_size
+              @max_match_length = [max_match_length, MAX_MATCH_LENGTH].min
+              @hash_table = {}
+            end
+
+            def find_match(data, position)
+              return nil if position >= data.size
+              return nil if data.size - position < MIN_MATCH_LENGTH
+
+              # Index all positions up to current if not done yet
+              ensure_indexed(data, position)
+
+              hash_val = hash_bytes(data, position)
+              candidates = @hash_table[hash_val] || []
+              best_match = find_best_among_candidates(data, position,
+                                                      candidates)
+              update_hash(hash_val, position)
+              best_match
+            end
+
+            def update(data, position)
+              return if position >= data.size
+
+              hash_val = hash_bytes(data, position)
+              update_hash(hash_val, position)
+            end
+
+            def reset
+              @hash_table.clear
+              @last_indexed = -1
+            end
+
+            def hash_chain_count
+              @hash_table.size
+            end
+
+            private
+
+            def ensure_indexed(data, position)
+              @last_indexed ||= -1
+              start_pos = [@last_indexed + 1, 0].max
+              (start_pos...position).each do |pos|
+                next if pos + MIN_MATCH_LENGTH > data.size
+
+                hash_val = hash_bytes(data, pos)
+                @hash_table[hash_val] ||= []
+                @hash_table[hash_val] << pos
+              end
+              @last_indexed = position - 1
+            end
+
+            def hash_bytes(data, position)
+              return 0 if position + 2 >= data.size
+
+              bytes = data.is_a?(String) ? data.bytes : data
+              (bytes[position] << 16) ^ (bytes[position + 1] << 8) ^ bytes[position + 2]
+            end
+
+            def find_best_among_candidates(data, position, candidates)
+              best_length = MIN_MATCH_LENGTH - 1
+              best_offset = 0
+              checked = 0
+
+              candidates.reverse_each do |candidate_pos|
+                offset = position - candidate_pos
+                break if offset > @window_size
+
+                checked += 1
+                break if checked > MAX_CHAIN_LENGTH
+
+                length = match_length(data, position, candidate_pos)
+                if length > best_length
+                  best_length = length
+                  best_offset = offset
+                  break if best_length >= @max_match_length
+                end
+              end
+
+              return nil if best_length < MIN_MATCH_LENGTH
+
+              Match.new(best_offset, best_length)
+            end
+
+            def match_length(data, pos1, pos2)
+              bytes = data.is_a?(String) ? data.bytes : data
+              max_len = [data.size - pos1, @max_match_length].min
+              length = 0
+              while length < max_len && bytes[pos1 + length] == bytes[pos2 + length]
+                length += 1
+              end
+              length
+            end
+
+            def update_hash(hash_val, position)
+              @hash_table[hash_val] ||= []
+              @hash_table[hash_val] << position
+              @hash_table[hash_val].shift if @hash_table[hash_val].size > MAX_CHAIN_LENGTH
+            end
+          end
+        end
+      end
+    end
+  end
+end