omnizip 0.3.1

data/lib/omnizip/algorithms/lzma/xz_match_finder_adapter.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+require_relative "match_finder"
+require_relative "constants"
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # XZ Utils-compatible match finder adapter
+      #
+      # Wraps existing MatchFinder to provide XZ Utils interface with:
+      # - Cursor-based position tracking
+      # - Multiple match finding (not just longest)
+      # - Skip and lookahead operations
+      #
+      # Based on: xz/src/liblzma/lz/lz_encoder_mf.c
+      class XzMatchFinderAdapter
+        include Constants
+
+        # Match structure matching XZ Utils
+        Match = Struct.new(:len, :dist, keyword_init: true) do
+          def to_s
+            "Match(len=#{len}, dist=#{dist})"
+          end
+        end
+
+        attr_reader :matches, :longest_len, :pos
+
+        # Initialize match finder adapter
+        #
+        # @param data [String, Array<Integer>] Input data
+        # @param dict_size [Integer] Dictionary size (default 8MB for XZ)
+        # @param nice_len [Integer] Nice match length (default 32)
+        def initialize(data, dict_size: 1 << 23, nice_len: 32)
+          @data = data.is_a?(String) ? data.bytes : data
+          @pos = 0
+          @dict_size = dict_size
+          @nice_len = nice_len
+
+          # Internal state
+          @matches = []
+          @longest_len = 0
+
+          # Hash table for match finding
+          @hash_table = {}
+        end
+
+        # Find all matches at current position
+        #
+        # Finds multiple matches of different lengths, not just the longest.
+        # Results stored in @matches array, longest length in @longest_len.
+        #
+        # @return [Integer] Longest match length (0 if no matches)
+        def find_matches
+          @matches.clear
+          @longest_len = 0
+
+          return 0 if @pos >= @data.size
+          return 0 if available < MATCH_LEN_MIN
+
+          # CRITICAL: Don't produce matches until there's enough data for decoder
+          # The decoder validates: dict_full > distance
+          # Where dict_full = decoded_byte_count (starting from 0)
+          # So for distance=N to be valid, we need at least N+1 bytes decoded
+          # We're at position @pos (0-based), so @pos bytes have been processed
+          # For distance=1 match: need @pos >= 2 (so decoder has dict_full=2)
+          # For distance=N match: need @pos >= N+1
+          # Simple check: Don't produce matches until @pos >= 2
+          return 0 if @pos < 2
+
+          # Find matches using hash chains
+          hash_val = compute_hash
+          positions = @hash_table[hash_val] || []
+
+          # Track best matches at each length
+          best_distances = {}
+
+          positions.reverse_each do |prev_pos|
+            distance = @pos - prev_pos
+            break if distance > @dict_size
+
+            # Skip self-matching (can happen when lookahead searches same position twice)
+            next if distance.zero?
+
+            match_len = calculate_match_length(prev_pos)
+            next if match_len < MATCH_LEN_MIN
+
+            # Keep best (shortest) distance for each length
+            if !best_distances[match_len] || distance < best_distances[match_len]
+              best_distances[match_len] = distance
+            end
+
+            # Update longest
+            @longest_len = match_len if match_len > @longest_len
+
+            # Stop if we found nice length
+            break if match_len >= @nice_len
+          end
+
+          # Convert to matches array (sorted by length)
+          best_distances.keys.sort.each do |len|
+            @matches << Match.new(len: len, dist: best_distances[len])
+          end
+
+          # Update hash table
+          update_hash(hash_val, @pos)
+
+          @longest_len
+        end
+
+        # Skip n bytes without finding matches
+        #
+        # Advances position and updates hash tables but doesn't search for matches.
+        # Used for rep matches where we already know what to encode.
+        #
+        # @param n [Integer] Number of bytes to skip
+        def skip(n)
+          n.times do
+            return if @pos >= @data.size
+
+            hash_val = compute_hash
+            update_hash(hash_val, @pos)
+            @pos += 1
+          end
+        end
+
+        # Move position forward by one byte
+        def move_pos
+          @pos += 1
+        end
+
+        # Bytes available from current position
+        #
+        # @return [Integer] Number of bytes remaining
+        def available
+          @data.size - @pos
+        end
+
+        # Get current byte at position
+        #
+        # @return [Integer, nil] Byte value or nil if at end
+        def current_byte
+          return nil if @pos >= @data.size
+
+          @data[@pos]
+        end
+
+        # Get byte at offset from current position
+        #
+        # @param offset [Integer] Offset from current position (can be negative)
+        # @return [Integer] Byte value (0 if out of bounds)
+        def get_byte(offset)
+          pos = @pos + offset
+          return 0 if pos.negative? || pos >= @data.size
+
+          @data[pos]
+        end
+
+        # Reset match finder to beginning
+        def reset
+          @pos = 0
+          @matches.clear
+          @longest_len = 0
+          @hash_table.clear
+        end
+
+        private
+
+        # Compute hash value for sequence at current position
+        #
+        # @return [Integer] Hash value
+        def compute_hash
+          return 0 if @pos + 2 >= @data.size
+
+          (@data[@pos] << 16) ^ (@data[@pos + 1] << 8) ^ @data[@pos + 2]
+        end
+
+        # Calculate match length between current position and previous position
+        #
+        # @param prev_pos [Integer] Previous position to compare against
+        # @return [Integer] Length of match
+        def calculate_match_length(prev_pos)
+          max_len = [available, @nice_len].min
+          length = 0
+
+          while length < max_len && @data[@pos + length] == @data[prev_pos + length]
+            length += 1
+          end
+
+          length
+        end
+
+        # Update hash table with position
+        #
+        # @param hash_val [Integer] Hash value
+        # @param pos [Integer] Position to add
+        def update_hash(hash_val, pos)
+          @hash_table[hash_val] ||= []
+          @hash_table[hash_val] << pos
+
+          # Keep hash chains from growing too large
+          @hash_table[hash_val].shift if @hash_table[hash_val].size > 1024
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/xz_price_calculator.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require_relative "constants"
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # XZ Utils-compatible price calculator
+      #
+      # Calculates the cost (in price units) of encoding symbols using
+      # probability models. Prices are based on logarithmic probabilities:
+      # price = -log2(probability) * scale_factor
+      #
+      # Uses precomputed tables for efficiency, matching XZ Utils exactly.
+      #
+      # Based on: xz/src/liblzma/rangecoder/price.h
+      class XzPriceCalculator
+        include Constants
+
+        # Price scale factor (matches XZ Utils)
+        PRICE_SHIFT_BITS = 4
+        PRICE_SCALE = 1 << PRICE_SHIFT_BITS
+
+        # BIT_MODEL_TOTAL = 2^11 = 2048 (from Constants, but define locally for clarity)
+        BIT_MODEL_TOTAL_LOCAL = 0x800
+        BIT_MODEL_TOTAL_BITS = 11
+
+        # Number of entries in price table
+        PRICE_TABLE_SIZE = BIT_MODEL_TOTAL_LOCAL >> PRICE_SHIFT_BITS
+
+        class << self
+          # Calculate price for encoding a single bit
+          #
+          # @param prob [Integer] Probability model value (0..BIT_MODEL_TOTAL)
+          # @param bit [Integer] Bit value (0 or 1)
+          # @return [Integer] Price in price units
+          def bit_price(prob, bit)
+            if bit.zero?
+              # Price for encoding 0
+              PRICE_TABLE[prob >> PRICE_SHIFT_BITS]
+            else
+              # Price for encoding 1
+              PRICE_TABLE[(BIT_MODEL_TOTAL_LOCAL - prob) >> PRICE_SHIFT_BITS]
+            end
+          end
+
+          # Calculate price for encoding a symbol using bit tree
+          #
+          # A bit tree encodes a symbol by encoding its bits from MSB to LSB,
+          # using probability models indexed by the partial symbol value.
+          #
+          # @param probs [Array<BitModel>] Probability models for tree
+          # @param num_bits [Integer] Number of bits in symbol
+          # @param symbol [Integer] Symbol value to encode
+          # @return [Integer] Total price in price units
+          def bittree_price(probs, num_bits, symbol)
+            price = 0
+            symbol |= (1 << num_bits) # Add sentinel bit
+
+            # Encode bits from MSB to LSB
+            (num_bits - 1).downto(0) do |i|
+              bit = (symbol >> i) & 1
+              model_idx = symbol >> (i + 1)
+              price += bit_price(probs[model_idx].probability, bit)
+            end
+
+            price
+          end
+
+          # Calculate price for encoding a symbol using reverse bit tree
+          #
+          # A reverse bit tree encodes a symbol by encoding its bits from
+          # LSB to MSB, used for distance encoding.
+          #
+          # @param probs [Array<BitModel>] Probability models for tree
+          # @param num_bits [Integer] Number of bits in symbol
+          # @param symbol [Integer] Symbol value to encode
+          # @return [Integer] Total price in price units
+          def bittree_reverse_price(probs, num_bits, symbol)
+            price = 0
+            model_idx = 1
+
+            # Encode bits from LSB to MSB
+            num_bits.times do |i|
+              bit = (symbol >> i) & 1
+              price += bit_price(probs[model_idx].probability, bit)
+              model_idx = (model_idx << 1) | bit
+            end
+
+            price
+          end
+
+          # Calculate price for direct bits (uniform distribution)
+          #
+          # Direct bits have no probability model, each bit costs the same.
+          #
+          # @param num_bits [Integer] Number of direct bits
+          # @return [Integer] Total price in price units
+          def direct_price(num_bits)
+            # Each direct bit costs 64 units (price of 0.5 probability)
+            num_bits << (PRICE_SHIFT_BITS + 2)
+          end
+
+          # Precompute logarithmic price table using Math.log2
+          #
+          # Generates a table mapping probabilities to prices using the formula:
+          # price[i] = -log2(i / BIT_MODEL_TOTAL) * PRICE_SCALE
+          #
+          # @return [Array<Integer>] Precomputed price table
+          def precompute_price_table
+            table = Array.new(PRICE_TABLE_SIZE)
+
+            PRICE_TABLE_SIZE.times do |i|
+              if i.zero?
+                # Handle zero probability case (maximum price)
+                table[i] = 15 << PRICE_SHIFT_BITS
+              else
+                # Reconstruct probability from table index
+                prob = (i << PRICE_SHIFT_BITS) + (PRICE_SCALE >> 1)
+                probability = prob.to_f / BIT_MODEL_TOTAL_LOCAL
+
+                # price = -log2(probability) * PRICE_SCALE
+                price = (-Math.log2(probability) * PRICE_SCALE).round
+                table[i] = price
+              end
+            end
+
+            table
+          end
+        end
+
+        # Precomputed logarithmic price table
+        # Each entry represents -log2(i/BIT_MODEL_TOTAL) * PRICE_SCALE
+        PRICE_TABLE = precompute_price_table.freeze
+
+        # Instance methods for convenience
+
+        # @param prob [Integer] Probability value
+        # @param bit [Integer] Bit value
+        # @return [Integer] Price
+        def bit_price(prob, bit)
+          self.class.bit_price(prob, bit)
+        end
+
+        # @param probs [Array<BitModel>] Probability models
+        # @param num_bits [Integer] Number of bits
+        # @param symbol [Integer] Symbol value
+        # @return [Integer] Price
+        def bittree_price(probs, num_bits, symbol)
+          self.class.bittree_price(probs, num_bits, symbol)
+        end
+
+        # @param probs [Array<BitModel>] Probability models
+        # @param num_bits [Integer] Number of bits
+        # @param symbol [Integer] Symbol value
+        # @return [Integer] Price
+        def bittree_reverse_price(probs, num_bits, symbol)
+          self.class.bittree_reverse_price(probs, num_bits, symbol)
+        end
+
+        # @param num_bits [Integer] Number of direct bits
+        # @return [Integer] Price
+        def direct_price(num_bits)
+          self.class.direct_price(num_bits)
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/xz_probability_models.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+require_relative "xz_buffered_range_encoder"
+require_relative "constants"
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # XZ Utils-compatible probability models
+      #
+      # Organizes all probability models used by LZMA encoder, matching
+      # XZ Utils structure exactly. All models start at probability 1024
+      # (BIT_MODEL_TOTAL / 2 = 0.5 probability).
+      #
+      # Uses XzBufferedRangeEncoder::Probability for mutable inline updates.
+      #
+      # Based on: xz/src/liblzma/lzma/lzma_encoder_private.h
+      class XzProbabilityModels
+        include Constants
+
+        # Literal context models
+        attr_reader :literal
+
+        # Match type models
+        attr_reader :is_match, :is_rep, :is_rep0, :is_rep1, :is_rep2
+        attr_reader :is_rep0_long
+
+        # Distance models
+        attr_reader :dist_slot, :dist_special, :dist_align
+
+        # Length encoders
+        attr_reader :match_len_encoder, :rep_len_encoder
+
+        # Initialize all probability models
+        #
+        # @param lc [Integer] Number of literal context bits (0-8)
+        # @param lp [Integer] Number of literal position bits (0-4)
+        # @param pb [Integer] Number of position bits (0-4)
+        def initialize(lc, lp, pb)
+          @lc = lc
+          @lp = lp
+          @pb = pb
+
+          init_literal_models(lc, lp)
+          init_match_models(pb)
+          init_distance_models
+          init_length_encoders(pb)
+        end
+
+        # Reset all probability models to initial state
+        def reset
+          reset_literal_models
+          reset_match_models
+          reset_distance_models
+          reset_length_encoders
+        end
+
+        private
+
+        # Initialize literal context models
+        # Ported from XZ Utils literal_init() in lzma_common.h
+        # Size: LITERAL_CODER_SIZE << (lc + lp) = 0x300 * (1 << (lc + lp))
+        # This is a FLAT array, not 2D, to match XZ Utils structure
+        def init_literal_models(lc, lp)
+          coders = 0x300 << (lc + lp)
+          @literal = Array.new(coders) { XzBufferedRangeEncoder::Probability.new }
+        end
+
+        # Initialize match type models
+        def init_match_models(pb)
+          num_pos_states = 1 << pb
+
+          # is_match[state][pos_state]
+          @is_match = Array.new(NUM_STATES) do
+            Array.new(num_pos_states) { XzBufferedRangeEncoder::Probability.new }
+          end
+
+          # is_rep[state]
+          @is_rep = Array.new(NUM_STATES) { XzBufferedRangeEncoder::Probability.new }
+
+          # is_rep0[state]
+          @is_rep0 = Array.new(NUM_STATES) { XzBufferedRangeEncoder::Probability.new }
+
+          # is_rep1[state]
+          @is_rep1 = Array.new(NUM_STATES) { XzBufferedRangeEncoder::Probability.new }
+
+          # is_rep2[state]
+          @is_rep2 = Array.new(NUM_STATES) { XzBufferedRangeEncoder::Probability.new }
+
+          # is_rep0_long[state][pos_state]
+          @is_rep0_long = Array.new(NUM_STATES) do
+            Array.new(num_pos_states) { XzBufferedRangeEncoder::Probability.new }
+          end
+        end
+
+        # Initialize distance models
+        def init_distance_models
+          # dist_slot[len_to_pos_state][dist_slot]
+          # len_to_pos_state: 0-3 (maps match length to state)
+          # dist_slot: 0-63 (6-bit distance slot)
+          @dist_slot = Array.new(NUM_LEN_TO_POS_STATES) do
+            Array.new(NUM_DIST_SLOTS) { XzBufferedRangeEncoder::Probability.new }
+          end
+
+          # dist_special[dist - 4] for slots 4-13 (160 models)
+          # Each slot has varying number of bits encoded with models
+          num_dist_special = NUM_FULL_DISTANCES - START_POS_MODEL_INDEX
+          @dist_special = Array.new(num_dist_special) { XzBufferedRangeEncoder::Probability.new }
+
+          # dist_align[align_bit] for alignment (16 models)
+          @dist_align = Array.new(DIST_ALIGN_SIZE) { XzBufferedRangeEncoder::Probability.new }
+        end
+
+        # Initialize length encoders
+        def init_length_encoders(pb)
+          @match_len_encoder = LengthEncoder.new(pb)
+          @rep_len_encoder = LengthEncoder.new(pb)
+        end
+
+        # Reset methods
+        def reset_literal_models
+          @literal.each { |prob| prob.value = BIT_MODEL_TOTAL >> 1 }
+        end
+
+        def reset_match_models
+          @is_match.each do |pos_states|
+            pos_states.each do |prob|
+              prob.value = BIT_MODEL_TOTAL >> 1
+            end
+          end
+          @is_rep.each { |prob| prob.value = BIT_MODEL_TOTAL >> 1 }
+          @is_rep0.each { |prob| prob.value = BIT_MODEL_TOTAL >> 1 }
+          @is_rep1.each { |prob| prob.value = BIT_MODEL_TOTAL >> 1 }
+          @is_rep2.each { |prob| prob.value = BIT_MODEL_TOTAL >> 1 }
+          @is_rep0_long.each do |pos_states|
+            pos_states.each do |prob|
+              prob.value = BIT_MODEL_TOTAL >> 1
+            end
+          end
+        end
+
+        def reset_distance_models
+          @dist_slot.each do |slots|
+            slots.each do |prob|
+              prob.value = BIT_MODEL_TOTAL >> 1
+            end
+          end
+          @dist_special.each { |prob| prob.value = BIT_MODEL_TOTAL >> 1 }
+          @dist_align.each { |prob| prob.value = BIT_MODEL_TOTAL >> 1 }
+        end
+
+        def reset_length_encoders
+          @match_len_encoder.reset
+          @rep_len_encoder.reset
+        end
+      end
+
+      # Length encoder with probability models and price tables
+      #
+      # Encodes match lengths using a 3-tier structure:
+      # - Low: lengths 2-9 (choice=0, 3 bits)
+      # - Mid: lengths 10-17 (choice=1, choice2=0, 3 bits)
+      # - High: lengths 18-273 (choice=1, choice2=1, 8 bits)
+      class LengthEncoder
+        include Constants
+
+        attr_reader :choice, :choice2, :low, :mid, :high, :prices, :counters
+
+        # Initialize length encoder
+        #
+        # @param pb [Integer] Number of position bits
+        def initialize(pb)
+          @pb = pb
+          @num_pos_states = 1 << pb
+
+          # Choice bits
+          @choice = XzBufferedRangeEncoder::Probability.new
+          @choice2 = XzBufferedRangeEncoder::Probability.new
+
+          # Low lengths (per position state)
+          @low = Array.new(@num_pos_states) do
+            Array.new(LEN_LOW_SYMBOLS) { XzBufferedRangeEncoder::Probability.new }
+          end
+
+          # Mid lengths (per position state)
+          @mid = Array.new(@num_pos_states) do
+            Array.new(LEN_MID_SYMBOLS) { XzBufferedRangeEncoder::Probability.new }
+          end
+
+          # High lengths (shared across position states)
+          @high = Array.new(LEN_HIGH_SYMBOLS) { XzBufferedRangeEncoder::Probability.new }
+
+          # Price tables (updated incrementally)
+          # prices[pos_state][length - MATCH_LEN_MIN]
+          @prices = Array.new(@num_pos_states) do
+            Array.new(MATCH_LEN_MAX - MATCH_LEN_MIN + 1, 0)
+          end
+
+          # Counters for price table updates
+          @counters = Array.new(@num_pos_states, 0)
+        end
+
+        # Reset all models to initial state
+        def reset
+          @choice.value = BIT_MODEL_TOTAL >> 1
+          @choice2.value = BIT_MODEL_TOTAL >> 1
+          @low.each do |models|
+            models.each do |prob|
+              prob.value = BIT_MODEL_TOTAL >> 1
+            end
+          end
+          @mid.each do |models|
+            models.each do |prob|
+              prob.value = BIT_MODEL_TOTAL >> 1
+            end
+          end
+          @high.each { |prob| prob.value = BIT_MODEL_TOTAL >> 1 }
+
+          # Reset price tables
+          @prices.each { |pos_prices| pos_prices.fill(0) }
+          @counters.fill(0)
+        end
+
+        # Get price for encoding length at position state
+        #
+        # @param pos_state [Integer] Position state (0 to 2^pb - 1)
+        # @param length [Integer] Match length (2 to 273)
+        # @return [Integer] Price in price units
+        def get_price(pos_state, length)
+          @prices[pos_state][length - MATCH_LEN_MIN]
+        end
+
+        # Set price for length at position state
+        #
+        # @param pos_state [Integer] Position state
+        # @param length [Integer] Match length
+        # @param price [Integer] Price value
+        def set_price(pos_state, length, price)
+          @prices[pos_state][length - MATCH_LEN_MIN] = price
+        end
+
+        # Decrement counter for position state
+        #
+        # @param pos_state [Integer] Position state
+        # @return [Boolean] True if counter reached zero
+        def decrement_counter(pos_state)
+          @counters[pos_state] -= 1
+          @counters[pos_state] <= 0
+        end
+
+        # Reset counter for position state
+        #
+        # @param pos_state [Integer] Position state
+        # @param value [Integer] Counter value
+        def reset_counter(pos_state, value)
+          @counters[pos_state] = value
+        end
+      end
+    end
+  end
+end