omnizip 0.4.7-arm-linux

data/lib/omnizip/algorithms/lzma/optimal_encoder.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # Optimal encoder - chooses best encoding
+      # Ported from lzma_encoder_optimum_fast.c
+      class OptimalEncoder
+        attr_reader :mode
+
+        def initialize(mode: :fast)
+          @mode = mode
+        end
+
+        # Find optimal encoding for current position
+        def find_optimal(position, match_finder, state, reps, models)
+          case @mode
+          when :fast
+            optimum_fast(position, match_finder, state, reps, models)
+          else
+            raise ArgumentError, "Unknown mode: #{@mode}"
+          end
+        end
+
+        private
+
+        # Fast mode: repeated-match scan plus first normal match. The
+        # unused state/models parameters keep the signature shared with
+        # a future normal (optimal-parse) mode.
+        def optimum_fast(position, match_finder, _state, reps, _models)
+          buf = match_finder.buffer # CRITICAL: Use match_finder.buffer, NOT dictionary.buffer!
+          buf_pos = position
+          buf_avail = [buf.bytesize - buf_pos, MATCH_LEN_MAX].min
+
+          # Not enough input for a match
+          return [UINT32_MAX, 1] if buf_avail < 2
+
+          # CRITICAL: Check repeated matches FIRST (before normal matches)
+          # This matches XZ Utils lzma_lzma_optimum_fast behavior
+          rep_len = 0
+          rep_index = 0
+
+          # For level 9, nice_len = MATCH_LEN_MAX = 273
+          nice_len = MATCH_LEN_MAX
+
+          # Look for repeated matches; scan the previous four match distances
+          reps.each_with_index do |rep_dist, i|
+            # Pointer to the beginning of the match candidate
+            # In XZ: buf_back = buf - coder->reps[i] - 1
+            # In Ruby: buf_back_index = buf_pos - rep_dist - 1
+            buf_back_index = buf_pos - rep_dist - 1
+
+            # Skip if out of bounds
+            next if buf_back_index.negative? || buf_back_index >= buf.bytesize
+
+            # If the first two bytes (2 == MATCH_LEN_MIN) do not match,
+            # this rep is not useful.
+            next if buf.getbyte(buf_pos) != buf.getbyte(buf_back_index) ||
+              buf.getbyte(buf_pos + 1) != buf.getbyte(buf_back_index + 1)
+
+            # The first two bytes matched. Calculate the length of the match.
+            # Use 64-bit integer comparison for speed (no string allocation)
+            max_match_len = [
+              buf.bytesize - buf_pos,
+              buf.bytesize - buf_back_index,
+              MATCH_LEN_MAX,
+              buf_avail,
+            ].min
+
+            len = 2
+            # Compare 8 bytes at a time using 64-bit integers read
+            # without intermediate strings (byteslice + unpack1
+            # allocates two Strings per step and dominated GC time).
+            while len + 8 <= max_match_len
+              v1 = read8(buf, buf_pos + len)
+              v2 = read8(buf, buf_back_index + len)
+              break if v1 != v2
+
+              len += 8
+            end
+            # Finalize with byte-by-byte for remaining bytes
+            while len < max_match_len &&
+                buf.getbyte(buf_pos + len) == buf.getbyte(buf_back_index + len)
+              len += 1
+            end
+
+            # If we have found a repeated match that is at least
+            # nice_len long, return it immediately.
+            if len >= nice_len
+              # Return rep index (0-3 for rep0-rep3)
+              return [i, len]
+            end
+
+            if len > rep_len
+              rep_index = i
+              rep_len = len
+            end
+          end
+
+          # Find normal matches from match finder
+          matches = match_finder.find_matches(position)
+
+          # We didn't find a long enough repeated match.
+          # Encode it as a normal match if the match length is at least nice_len
+          # AND the match distance stays within the data decoded so far.
+          # A match may reference anything before `position`: bytes already
+          # committed to the dictionary *and* bytes decoded earlier within
+          # the chunk currently being encoded.
+          best_normal = nil
+          matches.each do |m|
+            next if best_normal &&
+              (m.length < best_normal.length ||
+                (m.length == best_normal.length &&
+                  m.distance <= best_normal.distance))
+
+            best_normal = m
+          end
+
+          if best_normal && best_normal.length >= nice_len &&
+              best_normal.distance <= position
+            # Use normal match
+            return [best_normal.distance + REPS, best_normal.length]
+          end
+
+          # Compare best rep match vs best normal match
+          # Prefer longer match, for equal length prefer rep match (XZ behavior)
+          if rep_len >= 2 && rep_len >= (best_normal&.length || 0)
+            # Use repeated match
+            [rep_index, rep_len]
+          elsif best_normal && best_normal.distance <= position
+            # Use normal match (any length beats encoding literals)
+            [best_normal.distance + REPS, best_normal.length]
+          else
+            # Use literal - return UINT32_MAX to indicate literal (not 0!)
+            [UINT32_MAX, 1]
+          end
+        end
+
+        # Allocation-free little-endian 8-byte read.
+        def read8(buf, pos)
+          buf.getbyte(pos) |
+            (buf.getbyte(pos + 1) << 8) |
+            (buf.getbyte(pos + 2) << 16) |
+            (buf.getbyte(pos + 3) << 24) |
+            (buf.getbyte(pos + 4) << 32) |
+            (buf.getbyte(pos + 5) << 40) |
+            (buf.getbyte(pos + 6) << 48) |
+            (buf.getbyte(pos + 7) << 56)
+        end
+
+        # Constants
+        REPS = 4
+        MATCH_LEN_MAX = 273 # From lzma.h
+        MATCH_LEN_MIN = 2
+        UINT32_MAX = 0xFFFFFFFF
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/probability_models.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # Container for all LZMA probability models
+      # Ported from lzma_encoder_private.h
+      class ProbabilityModels
+        # Constants
+        STATES = 12
+        POS_STATES_MAX = 16
+        LEN_LOW_SYMBOLS = 8
+        LEN_MID_SYMBOLS = 8
+        LEN_HIGH_SYMBOLS = 16
+        LEN_SYMBOLS = 272
+        MATCH_LEN_MIN = 2
+        MATCH_LEN_MAX = 273
+        DIST_STATES = 4
+        DIST_SLOTS = 64
+        DIST_MODEL_START = 4
+        DIST_MODEL_END = 14
+        FULL_DISTANCES = 128
+        ALIGN_SIZE = 16
+
+        attr_accessor :literal, :is_match, :is_rep,
+                      :is_rep0, :is_rep1, :is_rep2, :is_rep0_long,
+                      :dist_slot, :dist_special, :dist_align,
+                      :match_len_encoder, :rep_len_encoder
+
+        def initialize(lc: 3, lp: 0, pb: 2)
+          @lc = lc
+          @lp = lp
+          @pb = pb
+
+          @literal = init_literal_models
+          @is_match = init_is_match_models
+          @is_rep = init_array(STATES, BitModel)
+          @is_rep0 = init_array(STATES, BitModel)
+          @is_rep1 = init_array(STATES, BitModel)
+          @is_rep2 = init_array(STATES, BitModel)
+          @is_rep0_long = init_is_rep0_long_models
+          @dist_slot = init_dist_slot_models
+          @dist_special = init_array(FULL_DISTANCES - DIST_MODEL_END, BitModel)
+          @dist_align = init_array(ALIGN_SIZE, BitModel)
+        end
+
+        private
+
+        def init_literal_models
+          num_contexts = 1 << (@lc + @lp)
+          Array.new(num_contexts * 0x300) { BitModel.new }
+        end
+
+        def init_is_match_models
+          Array.new(STATES * (1 << @pb)) { BitModel.new }
+        end
+
+        def init_is_rep0_long_models
+          Array.new(STATES * (1 << @pb)) { BitModel.new }
+        end
+
+        def init_dist_slot_models
+          Array.new(DIST_STATES * DIST_SLOTS) { BitModel.new }
+        end
+
+        def init_array(size, _model_class)
+          Array.new(size) { BitModel.new }
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/range_coder.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # Base class for range encoding and decoding
+      #
+      # This class provides the core arithmetic coding functionality
+      # used by both the encoder and decoder. Range coding is a form
+      # of arithmetic coding that uses integer arithmetic to represent
+      # probability ranges.
+      #
+      # The algorithm maintains a range [low, high) and subdivides it
+      # based on symbol probabilities. The range is periodically
+      # normalized to prevent arithmetic overflow/underflow.
+      class RangeCoder
+        include Constants
+
+        attr_reader :range, :low
+
+        # Initialize the range coder
+        #
+        # @param stream [IO] The input or output stream
+        def initialize(stream)
+          @stream = stream
+          @range = 0xFFFFFFFF
+          @low = 0
+        end
+
+        protected
+
+        # Normalize the range when it becomes too small
+        #
+        # This is an abstract method to be implemented by subclasses.
+        # The encoder and decoder have different normalization logic.
+        #
+        # @return [void]
+        def normalize
+          raise NotImplementedError,
+                "#{self.class} must implement #normalize"
+        end
+
+        # Create an array of bit models for probability management
+        #
+        # @param size [Integer] Number of bit models to create
+        # @return [Array<BitModel>] Array of initialized bit models
+        def create_bit_models(size)
+          Array.new(size) { BitModel.new }
+        end
+
+        # Get a bit model from an array based on index
+        #
+        # @param models [Array<BitModel>] Array of bit models
+        # @param index [Integer] Index to retrieve
+        # @return [BitModel] The bit model at the specified index
+        def get_bit_model(models, index)
+          models[index] ||= BitModel.new
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/range_decoder.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # Range decoder for LZMA decompression
+      #
+      # This class implements the decoding side of arithmetic coding
+      # using integer range arithmetic. It decodes bits from the
+      # compressed byte stream based on their probability models.
+      #
+      # The decoder mirrors the encoder's range subdivisions to
+      # extract the original bit values. It maintains a code value
+      # that represents the current position within the range.
+      class RangeDecoder < RangeCoder
+        attr_reader :code, :init_bytes_remaining, :range, :stream
+
+        # Initialize the range decoder
+        #
+        # @param input_stream [IO] The input stream of encoded bytes
+        def initialize(input_stream)
+          super
+          @code = 0
+          @initialization_complete = false
+          @init_bytes_remaining = 5
+          init_decoder
+        end
+
+        # Update the input stream (for LZMA2 multi-chunk streams)
+        #
+        # @param new_stream [IO] New input stream
+        # @return [void]
+        def update_stream(new_stream)
+          @stream = new_stream
+        end
+
+        # Decode a single bit using a probability model
+        #
+        # This is the hottest method (~5 billion calls for a 600MB file).
+        # normalize() and model.update() are inlined to eliminate method dispatch.
+        #
+        # @param model [BitModel] The probability model for this bit
+        # @return [Integer] The decoded bit value (0 or 1)
+        def decode_bit(model)
+          # Inline normalize: only the hot path (range < TOP check)
+          # Init bytes are handled eagerly in reset(), not here
+          if @range < 0x01000000
+            @range <<= 8
+            byte = @stream.getbyte
+            if byte.nil?
+              raise Omnizip::DecompressionError,
+                    "Truncated LZMA stream during range decoder normalization"
+            end
+
+            @code = ((@code << 8) | byte) & 0xFFFFFFFF
+          end
+
+          prob = model.probability
+          bound = (@range >> 11) * prob
+
+          if @code < bound
+            @range = bound
+            # Inline model.update(0): prob += (2048 - prob) >> 5
+            model.probability = prob + ((2048 - prob) >> 5)
+            0
+          else
+            @code -= bound
+            @range -= bound
+            # Inline model.update(1): prob -= prob >> 5
+            model.probability = prob - (prob >> 5)
+            1
+          end
+        end
+
+        # Decode bits directly without using probability model
+        #
+        # @param num_bits [Integer] Number of bits to decode
+        # @return [Integer] The decoded value
+        def decode_direct_bits(num_bits)
+          result = 0
+
+          num_bits.downto(1) do |_i|
+            # Inline normalize
+            if @range < 0x01000000
+              @range <<= 8
+              byte = @stream.getbyte
+              if byte.nil?
+                raise Omnizip::DecompressionError,
+                      "Truncated LZMA stream during range decoder normalization"
+              end
+
+              @code = ((@code << 8) | byte) & 0xFFFFFFFF
+            end
+            @range >>= 1
+
+            bit = @code >= @range ? 1 : 0
+
+            if bit == 1
+              @code -= @range
+              result = (result << 1) | 1
+            else
+              result = (result << 1) | 0
+            end
+          end
+
+          result
+        end
+
+        # Decode a cumulative frequency value (PPMd)
+        #
+        # @param total_freq [Integer] Total frequency of all symbols in context
+        # @return [Integer] The cumulative frequency value
+        def decode_freq(total_freq)
+          normalize
+          range_freq = @range / total_freq
+          @code / range_freq
+        end
+
+        # Normalize after decoding a symbol with frequency
+        #
+        # @param cum_freq [Integer] Cumulative frequency of decoded symbol
+        # @param freq [Integer] Frequency of decoded symbol
+        # @param total_freq [Integer] Total frequency of all symbols
+        # @return [void]
+        def normalize_freq(cum_freq, freq, total_freq)
+          range_freq = @range / total_freq
+          low_bound = range_freq * cum_freq
+          high_bound = range_freq * (cum_freq + freq)
+
+          @code -= low_bound
+          @range = (high_bound - low_bound) & 0xFFFFFFFF
+        end
+
+        # Decode bits directly using a base value (XZ Utils rc_direct pattern)
+        #
+        # @param num_bits [Integer] Number of bits to decode
+        # @param base [Integer] Base value to start from (2 or 3 for distances)
+        # @return [Integer] The decoded value
+        def decode_direct_bits_with_base(num_bits, base)
+          result = base
+          num_bits.times do |_i|
+            result = (result << 1) + 1
+            # Inline normalize
+            if @range < 0x01000000
+              @range <<= 8
+              byte = @stream.getbyte
+              if byte.nil?
+                raise Omnizip::DecompressionError,
+                      "Truncated LZMA stream during range decoder normalization"
+              end
+
+              @code = ((@code << 8) | byte) & 0xFFFFFFFF
+            end
+            @range >>= 1
+
+            bit = @code >= @range ? 1 : 0
+
+            if bit == 1
+              @code -= @range
+            else
+              result -= 1
+            end
+          end
+          result
+        end
+
+        # Reset the range decoder for a new chunk
+        #
+        # Resets state only. Call read_init_bytes after the stream is set
+        # to the correct input.
+        #
+        # @return [void]
+        def reset
+          @range = 0xFFFFFFFF
+          @code = 0
+          @init_bytes_remaining = 5
+        end
+
+        # Eagerly read the 5 initialization bytes from the current stream.
+        # Must be called after the stream is set to the correct input.
+        #
+        # @return [void]
+        def read_init_bytes
+          while @init_bytes_remaining.positive?
+            byte = @stream.getbyte
+            if byte.nil?
+              raise Omnizip::DecompressionError,
+                    "Truncated LZMA stream during range decoder initialization"
+            end
+
+            @code = ((@code << 8) | byte) & 0xFFFFFFFF
+            @init_bytes_remaining -= 1
+          end
+        end
+
+        # Normalize the range when it becomes too small
+        #
+        # Still needed for decode_freq() and other non-hot paths.
+        # The hot-path methods inline normalize directly.
+        #
+        # @return [void]
+        def normalize
+          # Handle lazy initialization if needed (for non-hot paths)
+          if @init_bytes_remaining.positive?
+            while @init_bytes_remaining.positive?
+              byte = @stream.getbyte
+              if byte.nil?
+                raise Omnizip::DecompressionError,
+                      "Truncated LZMA stream during range decoder initialization"
+              end
+
+              @code = ((@code << 8) | byte) & 0xFFFFFFFF
+              @init_bytes_remaining -= 1
+            end
+          end
+
+          if @range < TOP
+            byte = read_byte
+            @range <<= 8
+            @code = ((@code << 8) | byte) & 0xFFFFFFFF
+          end
+        end
+
+        private
+
+        # Initialize the decoder by reading the first bytes
+        #
+        # @return [void]
+        def init_decoder
+          5.times do
+            @code = ((@code << 8) | read_byte) & 0xFFFFFFFF
+            @init_bytes_remaining -= 1 if @init_bytes_remaining.positive?
+          end
+          @initialization_complete = true
+        end
+
+        # Read a single byte from the input stream
+        #
+        # @return [Integer] The byte value (0-255)
+        # @raise [Omnizip::DecompressionError] If stream is exhausted during normal decoding
+        def read_byte
+          byte = @stream.getbyte
+
+          if byte.nil? && @initialization_complete && @init_bytes_remaining.zero?
+            raise Omnizip::DecompressionError,
+                  "LZMA compressed data exhausted prematurely. The file may be corrupted or the uncompressed size field may be incorrect."
+          end
+
+          byte || 0
+        end
+      end
+    end
+  end
+end