omnizip 0.3.1

data/lib/omnizip/algorithms/lzma/xz_range_encoder.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # Range Encoder ported from XZ Utils range_encoder.c
+      #
+      # This class implements binary range coding, which is the core
+      # compression algorithm for LZMA. Range coding is a form of
+      # arithmetic coding that encodes bits into a compressed bitstream
+      # using probability models.
+      #
+      # The encoder maintains a range [low, low+range) and narrows this
+      # range as bits are encoded. When the range becomes too small, it
+      # is normalized and output bytes are produced.
+      #
+      # Ported from XZ Utils liblzma/range_encoder.c
+      class XZRangeEncoder
+        TOP_VALUE = 1 << 24
+        SHIFT_BITS = 8
+        BIT_MODEL_TOTAL_BITS = 11
+        BIT_MODEL_TOTAL = 1 << BIT_MODEL_TOTAL_BITS
+
+        attr_reader :cache, :range, :low
+
+        # Initialize a new range encoder
+        #
+        # @param output [IO] Output stream for compressed data
+        def initialize(output)
+          @output = output
+          @low = 0
+          @range = 0xFFFFFFFF
+          @cache = 0
+          @cache_size = 1
+        end
+
+        # Encode a single bit using probability model
+        #
+        # This method encodes a bit (0 or 1) using an adaptive probability
+        # model. The probability model is updated after encoding to adapt
+        # to the input data statistics.
+        #
+        # Ported from XZ Utils range_encoder.c rc_bit()
+        #
+        # @param model [BitModel] Probability model for this bit
+        # @param bit [Integer] Bit value to encode (0 or 1)
+        # @return [void]
+        def encode_bit(model, bit)
+          prob = model.probability
+          bound = (@range >> BIT_MODEL_TOTAL_BITS) * prob
+
+          if bit.zero?
+            @range = bound
+          else
+            @low += bound
+            @range -= bound
+          end
+
+          normalize! if @range < TOP_VALUE
+
+          # Update probability model based on bit value
+          # Matches decoder behavior (proper OOP symmetry)
+          model.update(bit)
+        end
+
+        # Encode multiple bits as a bittree
+        #
+        # A bittree is a binary tree where each node has a probability model.
+        # This method encodes a value by traversing the tree from the root,
+        # encoding the bit at each node and following the corresponding branch.
+        #
+        # Ported from XZ Utils range_encoder.c rc_bittree()
+        #
+        # @param models [Array<BitModel>] Array of probability models for tree nodes
+        # @param num_bits [Integer] Number of bits to encode
+        # @param value [Integer] Value to encode
+        # @return [void]
+        def encode_bittree(models, num_bits, value)
+          index = 1
+          bit = num_bits - 1
+
+          while bit >= 0
+            bit_value = (value >> bit) & 1
+            encode_bit(models[index - 1], bit_value)
+            index = (index << 1) | bit_value
+            bit -= 1
+          end
+        end
+
+        # Encode multiple bits as a reverse bittree
+        #
+        # Similar to encode_bittree but processes bits in reverse order.
+        # This is used for certain LZMA encoding operations.
+        #
+        # Ported from XZ Utils range_encoder.c rc_bittree_reverse()
+        #
+        # @param models [Array<BitModel>] Array of probability models for tree nodes
+        # @param num_bits [Integer] Number of bits to encode
+        # @param value [Integer] Value to encode
+        # @return [void]
+        def encode_bittree_reverse(models, num_bits, value)
+          index = 1
+          bit = 0
+
+          while bit < num_bits
+            bit_value = (value >> bit) & 1
+            encode_bit(models[index - 1], bit_value)
+            index = (index << 1) | bit_value
+            bit += 1
+          end
+        end
+
+        # Encode a direct bit (without probability model)
+        #
+        # This method encodes a bit with fixed 0.5 probability.
+        # Used for encoding values where no adaptive model is available.
+        #
+        # Ported from XZ Utils range_encoder.c rc_direct()
+        #
+        # @param value [Integer] Value to encode (0 or 1)
+        # @return [void]
+        def encode_direct(value)
+          @range >>= 1
+          @low += @range if value != 0
+          normalize! if @range < TOP_VALUE
+        end
+
+        # Flush pending data to output stream
+        #
+        # This method flushes any remaining data in the range encoder
+        # to the output stream. This must be called before the encoder
+        # is discarded.
+        #
+        # Ported from XZ Utils range_encoder.c rc_flush()
+        #
+        # @return [void]
+        def flush!
+          (5 - @cache_size).times do
+            shift_low
+          end
+        end
+
+        private
+
+        # Normalize the range encoder state
+        #
+        # When the range becomes too small (< TOP_VALUE), it needs to be
+        # normalized by shifting left and outputting bytes as needed.
+        #
+        # Ported from XZ Utils range_encoder.c rc_normalize()
+        #
+        # @return [void]
+        def normalize!
+          if @range < TOP_VALUE
+            @range <<= SHIFT_BITS
+            shift_low
+          end
+        end
+
+        # Shift low value and output bytes as needed
+        #
+        # This method handles the carry propagation and byte output
+        # for the range encoder. When the high byte of low changes,
+        # it outputs bytes to the stream.
+        #
+        # Ported from XZ Utils range_encoder.c rc_shift_low()
+        # See: /Users/mulgogi/src/external/xz/src/liblzma/rangecoder/range_encoder.h:140-186
+        #
+        # @return [void]
+        def shift_low
+          # Extract low 32 bits and high 32 bits (carry)
+          # XZ Utils: if ((uint32_t)(rc->low) < (uint32_t)(0xFF000000) || (uint32_t)(rc->low >> 32) != 0)
+          # This condition is TRUE when:
+          #   - low32 < 0xFF000000 (the high byte of low is NOT 0xFF)
+          #   - OR high != 0 (there's a carry from the low value)
+          # When TRUE: write output bytes
+          # When FALSE: increment cache_size (we're in a run of 0xFF bytes)
+          low32 = @low & 0xFFFFFFFF
+          high = (@low >> 32) & 0xFF
+
+          if low32 < 0xFF000000 || high != 0
+            # Write pending cache bytes (with carry if present)
+            temp = @cache
+            while @cache_size.positive?
+              @output.putc((temp + high) & 0xFF)
+              temp = 0xFF
+              @cache_size -= 1
+            end
+            # Update cache to the high byte of low
+            @cache = (low32 >> 24) & 0xFF
+          else
+            # High byte of low is 0xFF and no carry - increment pending count
+            @cache_size += 1
+          end
+
+          # Shift low left by 8 bits (keeping only low 24 bits before shift)
+          # XZ Utils: low = (low & 0x00FFFFFF) << RC_SHIFT_BITS;
+          @low = (low32 & 0x00FFFFFF) << SHIFT_BITS
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/xz_range_encoder_exact.rb

@@ -0,0 +1,331 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Direct port of XZ Utils range encoder to Ruby
+# Based on: xz/src/liblzma/rangecoder/range_encoder.h
+
+require_relative "constants"
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # XZ Utils-compatible range encoder (direct port)
+      #
+      # This is a direct port of the XZ Utils range encoder implementation
+      # to ensure exact algorithmic compatibility with XZ Utils output.
+      class XzRangeEncoder
+        include Constants
+
+        # Range encoder constants (matching XZ Utils range_common.h)
+        SHIFT_BITS = 8        # RC_SHIFT_BITS
+        TOP_BITS = 24         # RC_TOP_BITS
+        TOP = 0x01000000      # 2^24
+        BIT_MODEL_TOTAL_BITS = 11
+        BIT_MODEL_TOTAL = 2048 # 2^11
+
+        # Symbol types (matching XZ Utils enum)
+        RC_BIT_0 = 0
+        RC_BIT_1 = 1
+        RC_DIRECT_0 = 2
+        RC_DIRECT_1 = 3
+        RC_FLUSH = 4
+
+        # Maximum symbols that can be queued
+        RC_SYMBOLS_MAX = 53
+
+        attr_reader :out_total, :count, :low, :range, :cache, :cache_size
+
+        # Initialize the range encoder
+        #
+        # @param output_stream [IO] The output stream for encoded bytes
+        def initialize(output_stream)
+          @stream = output_stream
+          # Initialize @out_total BEFORE calling reset
+          @out_total = 0
+          reset
+        end
+
+        # Reset encoder to initial state (matches XZ Utils rc_reset)
+        def reset
+          @low = 0
+          @cache_size = 1 # CRITICAL: XZ starts with 1, not 0
+          @range = 0xFFFFFFFF
+          @cache = 0
+          # CRITICAL: Reset @out_total to match XZ Utils behavior (line 63 of range_encoder.h)
+          # This ensures bytes_for_decode returns the correct count
+          @out_total = 0
+          # NOTE: @pre_flush_out_total is NOT reset - it retains its value for bytes_for_decode
+          # It will be reset to 0 when a new chunk starts (via initialize)
+          @count = 0
+          @pos = 0
+          @symbols = Array.new(RC_SYMBOLS_MAX, 0)
+          @probs = Array.new(RC_SYMBOLS_MAX, nil)
+        end
+
+        # Forget pending symbols (matches XZ Utils rc_forget)
+        def forget
+          raise "Cannot forget while encoding" if @pos != 0
+
+          @count = 0
+        end
+
+        # Queue a bit for encoding (matches XZ Utils rc_bit)
+        #
+        # @param prob [Probability] Probability model
+        # @param bit [Integer] Bit value (0 or 1)
+        def bit(prob, bit)
+          raise "Symbol buffer overflow" if @count >= RC_SYMBOLS_MAX
+
+          @symbols[@count] = bit
+          @probs[@count] = prob
+          @count += 1
+        end
+        alias queue_bit bit
+
+        # Queue bittree encoding (matches XZ Utils rc_bittree)
+        #
+        # @param probs [Array<Probability>] Probability array
+        # @param bit_count [Integer] Number of bits
+        # @param symbol [Integer] Symbol to encode
+        def bittree(probs, bit_count, symbol)
+          model_index = 1
+
+          bit_count.times do
+            bit = (symbol >> (bit_count -= 1)) & 1
+            bit(probs[model_index], bit)
+            model_index = (model_index << 1) | bit
+          end
+        end
+
+        # Queue bittree encoding in reverse (matches XZ Utils rc_bittree_reverse)
+        #
+        # @param probs [Array<Probability>] Probability array
+        # @param bit_count [Integer] Number of bits
+        # @param symbol [Integer] Symbol to encode
+        def bittree_reverse(probs, bit_count, symbol)
+          model_index = 1
+
+          bit_count.times do
+            bit = symbol & 1
+            symbol >>= 1
+            bit(probs[model_index], bit)
+            model_index = (model_index << 1) | bit
+          end
+        end
+
+        # Queue direct bits (matches XZ Utils rc_direct)
+        #
+        # @param value [Integer] Value to encode
+        # @param bit_count [Integer] Number of bits
+        def direct(value, bit_count)
+          bit_count.times do
+            raise "Symbol buffer overflow" if @count >= RC_SYMBOLS_MAX
+
+            @symbols[@count] = RC_DIRECT_0 | ((value >> (bit_count -= 1)) & 1)
+            @probs[@count] = nil
+            @count += 1
+          end
+        end
+
+        # Queue flush operation (matches XZ Utils rc_flush)
+        def flush
+          puts "[FLUSH] Adding 5 RC_FLUSH symbols, @count before=#{@count}" if ENV["DEBUG"]
+          5.times do
+            raise "Symbol buffer overflow" if @count >= RC_SYMBOLS_MAX
+
+            @symbols[@count] = RC_FLUSH
+            @probs[@count] = nil
+            @count += 1
+          end
+          puts "[FLUSH] @count after=#{@count}" if ENV["DEBUG"]
+        end
+        alias queue_flush flush
+
+        # Get number of pending bytes (matches XZ Utils rc_pending)
+        #
+        # @return [Integer] Number of pending output bytes
+        def pending
+          @cache_size + 5 - 1
+        end
+
+        # Check if no symbols are queued
+        #
+        # @return [Boolean] True if no symbols queued
+        def none?
+          @count.zero?
+        end
+
+        # Encode all queued symbols to output (matches XZ Utils rc_encode)
+        #
+        # @param out [IO,String] Output buffer
+        # @param out_pos [IntegerRef] Current output position
+        # @param out_size [Integer] Output buffer size
+        # @return [Boolean] True if output buffer filled before encoding complete
+        def encode(out, out_pos, out_size)
+          raise "Symbol buffer overflow" if @count > RC_SYMBOLS_MAX
+
+          puts "[ENCODE] Start: @count=#{@count} @pos=#{@pos} @out_total=#{@out_total}" if ENV["DEBUG"]
+
+          skip_increment = false
+
+          while @pos < @count
+            # Normalize (matches XZ Utils exactly)
+            if @range < TOP
+              return true if shift_low(out, out_pos, out_size)
+
+              @range <<= SHIFT_BITS
+            end
+
+            # Encode current symbol
+            case @symbols[@pos]
+            when RC_BIT_0
+              prob = @probs[@pos]
+              # XZ Utils: rc->range = (rc->range >> RC_BIT_MODEL_TOTAL_BITS) * prob
+              @range = (@range >> BIT_MODEL_TOTAL_BITS) * prob.value
+              # XZ Utils: prob += (RC_BIT_MODEL_TOTAL - prob) >> RC_MOVE_BITS
+              prob.value += (BIT_MODEL_TOTAL - prob.value) >> MOVE_BITS
+              @probs[@pos] = prob
+
+            when RC_BIT_1
+              prob = @probs[@pos]
+              # XZ Utils: bound = prob * (rc->range >> RC_BIT_MODEL_TOTAL_BITS)
+              bound = prob.value * (@range >> BIT_MODEL_TOTAL_BITS)
+              @low += bound
+              @range -= bound
+              # XZ Utils: prob -= prob >> RC_MOVE_BITS
+              prob.value -= prob.value >> MOVE_BITS
+              @probs[@pos] = prob
+
+            when RC_DIRECT_0
+              @range >>= 1
+
+            when RC_DIRECT_1
+              @range >>= 1
+              @low += @range
+
+            when RC_FLUSH
+              # Prevent further normalizations (XZ Utils behavior)
+              @range = 0xFFFFFFFF
+
+              puts "[ENCODE] RC_FLUSH: @pos=#{@pos} @count=#{@count}" if ENV["DEBUG"]
+
+              iteration = 0
+              # Flush the last five bytes (see rc_flush)
+              begin
+                iteration += 1
+                puts "[ENCODE] RC_FLUSH iteration #{iteration}: @pos=#{@pos}" if ENV["DEBUG"]
+                return true if shift_low(out, out_pos, out_size)
+
+                puts "[ENCODE] After shift_low: @pos=#{@pos}" if ENV["DEBUG"]
+              end while (@pos += 1) < @count
+
+              puts "[ENCODE] After RC_FLUSH loop: total #{iteration} iterations" if ENV["DEBUG"]
+
+              # Reset the range encoder (matches XZ Utils)
+              reset
+              # CRITICAL: Skip the @pos increment at loop end because do-while already did it
+              skip_increment = true
+              break
+
+            else
+              raise "Unknown symbol type: #{@symbols[@pos]}"
+            end
+
+            @pos += 1 unless skip_increment
+          end
+
+          puts "[ENCODE] End: @count=#{@count} @pos=#{@pos} @out_total=#{@out_total}" if ENV["DEBUG"]
+
+          @count = 0
+          @pos = 0
+
+          false
+        end
+
+        # Shift low bytes to output (matches XZ Utils rc_shift_low)
+        #
+        # @param out [IO,String] Output buffer
+        # @param out_pos [IntegerRef] Current output position
+        # @param out_size [Integer] Output buffer size
+        # @return [Boolean] True if output buffer filled
+        def shift_low(out, out_pos, out_size)
+          # XZ Utils: if ((uint32_t)(rc->low) < (uint32_t)(0xFF000000) || (uint32_t)(rc->low >> 32) != 0)
+          if (@low & 0xFFFFFFFF) < 0xFF000000 || (@low >> 32) != 0
+            # XZ Utils: do { ... } while (--rc->cache_size != 0);
+            while @cache_size.positive?
+              return true if out_pos.value == out_size
+
+              # XZ Utils: out[*out_pos] = rc->cache + (uint8_t)(rc->low >> 32)
+              output_byte = @cache + ((@low >> 32) & 0xFF)
+
+              if out.is_a?(String)
+                out.setbyte(out_pos.value, output_byte)
+              else
+                out.putc(output_byte)
+              end
+
+              out_pos.value += 1
+              @out_total += 1
+              @cache = 0xFF
+
+              @cache_size -= 1
+            end
+
+            # XZ Utils: rc->cache = (rc->low >> 24) & 0xFF
+            @cache = (@low >> 24) & 0xFF
+          end
+
+          # XZ Utils: ++rc->cache_size; rc->low = (rc->low & 0x00FFFFFF) << RC_SHIFT_BITS
+          @cache_size += 1
+          @low = (@low & 0x00FFFFFF) << SHIFT_BITS
+
+          false
+        end
+
+        # Adapter method: alias for bit (to match existing API)
+        alias queue_bit bit
+
+        # Adapter method: alias for bittree (to match existing API)
+        alias queue_bittree bittree
+
+        # Adapter method: alias for bittree_reverse (to match existing API)
+        alias queue_bittree_reverse bittree_reverse
+
+        # Adapter method: alias for direct (to match existing API)
+        def queue_direct_bits(value, num_bits)
+          direct(value, num_bits)
+        end
+
+        # Adapter method: alias for encode (to match existing API)
+        alias encode_symbols encode
+
+        # Adapter method: match existing API
+        alias queue_flush flush
+
+        # Forget pending symbols (matches XZ Utils rc_forget)
+        def forget
+          raise "Cannot forget while encoding" if @pos != 0
+
+          @count = 0
+        end
+
+        # Calculate pending output bytes
+        #
+        # @return [Integer] Number of bytes decoder will consume
+        def bytes_for_decode
+          @out_total
+        end
+      end
+
+      # Reference wrapper for integer (for out_pos parameter)
+      class IntRef
+        attr_accessor :value
+
+        def initialize(val)
+          @value = val
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/xz_state.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # XZ Utils-compatible LZMA state machine
+      #
+      # Tracks encoding context via 12-state machine to predict
+      # optimal probability models for upcoming symbols.
+      #
+      # Based on: xz/src/liblzma/lzma/lzma_common.h
+      class XzState
+        # 12 LZMA states (matching XZ Utils exactly)
+        STATE_LIT_LIT = 0           # literal after literal
+        STATE_MATCH_LIT_LIT = 1     # literal after literal after match
+        STATE_REP_LIT_LIT = 2       # literal after literal after rep
+        STATE_SHORTREP_LIT_LIT = 3  # literal after literal after shortrep
+        STATE_MATCH_LIT = 4         # literal after match
+        STATE_REP_LIT = 5           # literal after rep
+        STATE_SHORTREP_LIT = 6      # literal after shortrep
+        STATE_LIT_MATCH = 7         # match after literal
+        STATE_LIT_LONGREP = 8       # longrep after literal
+        STATE_LIT_SHORTREP = 9      # shortrep after literal
+        STATE_NONLIT_MATCH = 10     # match after non-literal
+        STATE_NONLIT_REP = 11       # rep after non-literal
+
+        LIT_STATES = 7 # States 0-6 indicate previous was literal
+
+        attr_accessor :value
+
+        def initialize(initial = STATE_LIT_LIT)
+          @value = initial
+        end
+
+        # Update state after encoding literal
+        # Matches XZ Utils update_literal() macro
+        def update_literal
+          old_value = @value
+          @value = if @value <= STATE_SHORTREP_LIT_LIT
+                     STATE_LIT_LIT
+                   elsif @value <= STATE_LIT_SHORTREP
+                     @value - 3
+                   else
+                     @value - 6
+                   end
+          if ENV["LZMA_DEBUG"]
+            warn "DEBUG: update_literal - state: #{old_value} → #{@value}"
+          end
+        end
+
+        # Update state after encoding match
+        # Matches XZ Utils update_match() macro
+        def update_match
+          old_value = @value
+          @value = @value < LIT_STATES ? STATE_LIT_MATCH : STATE_NONLIT_MATCH
+          if ENV["LZMA_DEBUG"]
+            warn "DEBUG: update_match - state: #{old_value} → #{@value}"
+          end
+        end
+
+        # Update state after encoding long rep match
+        # Matches XZ Utils update_long_rep() macro
+        def update_long_rep
+          @value = @value < LIT_STATES ? STATE_LIT_LONGREP : STATE_NONLIT_REP
+        end
+
+        # Update state after encoding short rep (1 byte)
+        # Matches XZ Utils update_short_rep() macro
+        def update_short_rep
+          @value = @value < LIT_STATES ? STATE_LIT_SHORTREP : STATE_NONLIT_REP
+        end
+
+        # Check if previous symbol was literal
+        def literal_state?
+          @value < LIT_STATES
+        end
+
+        # Create a copy of this state
+        def dup
+          XzState.new(@value)
+        end
+
+        # Reset to initial state
+        def reset
+          @value = STATE_LIT_LIT
+        end
+
+        # Check if state is valid
+        def valid?
+          @value.between?(STATE_LIT_LIT, STATE_NONLIT_REP)
+        end
+
+        # String representation for debugging
+        def to_s
+          STATE_NAMES[@value] || "INVALID(#{@value})"
+        end
+
+        # State names for debugging
+        STATE_NAMES = {
+          STATE_LIT_LIT => "STATE_LIT_LIT",
+          STATE_MATCH_LIT_LIT => "STATE_MATCH_LIT_LIT",
+          STATE_REP_LIT_LIT => "STATE_REP_LIT_LIT",
+          STATE_SHORTREP_LIT_LIT => "STATE_SHORTREP_LIT_LIT",
+          STATE_MATCH_LIT => "STATE_MATCH_LIT",
+          STATE_REP_LIT => "STATE_REP_LIT",
+          STATE_SHORTREP_LIT => "STATE_SHORTREP_LIT",
+          STATE_LIT_MATCH => "STATE_LIT_MATCH",
+          STATE_LIT_LONGREP => "STATE_LIT_LONGREP",
+          STATE_LIT_SHORTREP => "STATE_LIT_SHORTREP",
+          STATE_NONLIT_MATCH => "STATE_NONLIT_MATCH",
+          STATE_NONLIT_REP => "STATE_NONLIT_REP",
+        }.freeze
+      end
+    end
+  end
+end