omnizip 0.3.1

data/lib/omnizip/implementations/seven_zip/lzma/match_finder.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+require_relative "../../../algorithms/lzma/constants"
+require_relative "../../../algorithms/lzma/match_finder_config"
+
+module Omnizip
+  module Implementations
+    module SevenZip
+      module LZMA
+        # 7-Zip LZMA SDK match finder implementation.
+        #
+        # This is the original SdkMatchFinder moved from algorithms/lzma/sdk_match_finder.rb
+        # to the new namespace structure.
+        #
+        # Ported from 7-Zip LZMA SDK by Igor Pavlov.
+        class MatchFinder
+          include Omnizip::Algorithms::LZMA::Constants
+
+          # Represents a match found in the dictionary
+          class Match
+            attr_reader :length, :distance
+
+            def initialize(length, distance)
+              @length = length
+              @distance = distance
+            end
+          end
+
+          attr_reader :config
+
+          # Initialize the SDK-compatible match finder
+          #
+          # @param config [MatchFinderConfig] Configuration object
+          def initialize(config)
+            @config = config
+            @window_size = config.window_size
+            @max_match_length = config.max_match_length
+            @chain_length = config.chain_length
+            @lazy_matching = config.lazy_matching
+
+            # Hash table: maps hash value to position
+            # SDK uses separate hash2 and hash3 tables, but we simplify
+            # to single hash table with chaining
+            @hash_table = {}
+
+            # Hash chain: stores previous positions for each hash value
+            @hash_chain = {}
+
+            # CRC table for hash computation (SDK uses CRC)
+            init_crc_table
+          end
+
+          # Find the longest match at the given position
+          #
+          # Implements SDK's GetMatches() function from LzFind.c
+          #
+          # @param data [String, Array<Integer>] Input data
+          # @param pos [Integer] Current position in data
+          # @return [Match, nil] Best match or nil if no match found
+          def find_longest_match(data, pos)
+            return nil if pos >= data.size
+            return nil if data.size - pos < MATCH_LEN_MIN
+
+            if @lazy_matching && @lazy_match
+              # Return lazy match from previous position
+              match = @lazy_match
+              @lazy_match = nil
+              # Don't update hash - current position was already added when lazy match was created
+              return match
+            end
+
+            best_match = find_best_match(data, pos)
+
+            if @lazy_matching && best_match && pos + 1 < data.size
+              # Try next position for potentially better match
+              next_match = find_best_match(data, pos + 1)
+              if next_match && next_match.length > best_match.length
+                # Save better match for next call
+                @lazy_match = next_match
+                # Don't update hash - we'll add it when lazy match is consumed
+                return nil
+              end
+            end
+
+            # CRITICAL: Update hash AFTER finding matches
+            # This ensures the current position is available for future matches
+            update_hash(data, pos)
+            best_match
+          end
+
+          # Reset the match finder state
+          #
+          # @return [void]
+          def reset
+            @hash_table.clear
+            @hash_chain.clear
+            @lazy_match = nil
+          end
+
+          private
+
+          # Find best match at position (SDK's GetMatches core logic)
+          #
+          # Searches both 2-byte and 3-byte hash chains for the best match.
+          #
+          # @param data [String, Array<Integer>] Input data
+          # @param pos [Integer] Current position
+          # @return [Match, nil] Best match or nil
+          def find_best_match(data, pos)
+            best_match = nil
+            best_length = MATCH_LEN_MIN - 1
+
+            hashes = compute_hashes(data, pos)
+            return nil if hashes.empty?
+
+            # Search both hash chains
+            hashes.each_value do |hash_val|
+              positions = @hash_chain[hash_val] || []
+              next if positions.empty?
+
+              # SDK traverses hash chain from most recent to oldest
+              # Limited by chain_length (nice_len in SDK)
+              count = 0
+              positions.reverse_each do |prev_pos|
+                break if count >= @chain_length
+                break if pos <= prev_pos || pos - prev_pos > @window_size
+
+                match_len = calculate_match_length(data, pos, prev_pos)
+
+                if match_len > best_length
+                  best_length = match_len
+                  best_match = Match.new(match_len, pos - prev_pos)
+
+                  # SDK optimization: stop if we found max length
+                  break if best_length >= @max_match_length
+                end
+
+                count += 1
+              end
+
+              # If we found a full-length match, no need to check other hashes
+              break if best_length >= @max_match_length
+            end
+
+            best_match
+          end
+
+          # Compute hash value using SDK's algorithm
+          #
+          # SDK uses CRC-based hashing with multiple hash levels:
+          # - hash2: 2-byte hash
+          # - hash3: 3-byte hash
+          # - hash4: 4-byte hash (binary tree mode)
+          #
+          # We compute both 2-byte and 3-byte hashes and store matches
+          # in both hash tables to ensure matches are found regardless
+          # of which hash size is used at query time.
+          #
+          # @param data [String, Array<Integer>] Input data
+          # @param pos [Integer] Position to hash from
+          # @return [Integer] Hash value (3-byte if available, else 2-byte)
+          def compute_hash(data, pos)
+            bytes = data.is_a?(String) ? data.bytes : data
+
+            if pos + 3 <= data.size
+              # 3-byte hash: CRC[byte[0]] ^ byte[1] ^ (byte[2] << 8)
+              hash = @crc_table[bytes[pos]] ^ bytes[pos + 1]
+              hash ^= (bytes[pos + 2] << 8)
+              hash & 0xFFFF
+            elsif pos + 2 <= data.size
+              # 2-byte hash: CRC[byte[0]] ^ byte[1]
+              hash = @crc_table[bytes[pos]] ^ bytes[pos + 1]
+              hash & 0xFFFF
+            end
+            # Less than 2 bytes remaining returns nil implicitly
+          end
+
+          # Compute both 2-byte and 3-byte hashes
+          #
+          # @param data [String, Array<Integer>] Input data
+          # @param pos [Integer] Position to hash from
+          # @return [Array<Integer>] Array of [hash2, hash3] or [hash2, nil]
+          def compute_hashes(data, pos)
+            bytes = data.is_a?(String) ? data.bytes : data
+            hashes = {}
+
+            # 2-byte hash (always compute if possible)
+            if pos + 2 <= data.size
+              hash2 = @crc_table[bytes[pos]] ^ bytes[pos + 1]
+              hashes[:hash2] = hash2 & 0xFFFF
+            end
+
+            # 3-byte hash (only if 3+ bytes available)
+            if pos + 3 <= data.size
+              hash3 = @crc_table[bytes[pos]] ^ bytes[pos + 1]
+              hash3 ^= (bytes[pos + 2] << 8)
+              hashes[:hash3] = hash3 & 0xFFFF
+            end
+
+            hashes
+          end
+
+          # Calculate match length between two positions
+          #
+          # SDK compares bytes until mismatch or max length
+          #
+          # @param data [String, Array<Integer>] Input data
+          # @param pos1 [Integer] First position
+          # @param pos2 [Integer] Second position
+          # @return [Integer] Length of match
+          def calculate_match_length(data, pos1, pos2)
+            bytes = data.is_a?(String) ? data.bytes : data
+            max_len = [data.size - pos1, @max_match_length].min
+            length = 0
+
+            while length < max_len && bytes[pos1 + length] == bytes[pos2 + length]
+              length += 1
+            end
+
+            length
+          end
+
+          # Update hash table with new position
+          #
+          # Stores position in both 2-byte and 3-byte hash chains
+          # to ensure matches are found regardless of hash size used at query time.
+          #
+          # @param data [String, Array<Integer>] Input data
+          # @param pos [Integer] Position to add
+          # @return [void]
+          def update_hash(data, pos)
+            hashes = compute_hashes(data, pos)
+            return if hashes.empty?
+
+            hashes.each_value do |hash_val|
+              @hash_chain[hash_val] ||= []
+              @hash_chain[hash_val] << pos
+
+              # Keep hash chains from growing too large
+              # SDK uses cyclic buffer, we use simple truncation
+              max_chain = @chain_length * 2
+              @hash_chain[hash_val].shift if @hash_chain[hash_val].size > max_chain
+            end
+          end
+
+          # Initialize CRC table for hash computation
+          #
+          # SDK uses CRC32 table for hashing
+          #
+          # @return [void]
+          def init_crc_table
+            @crc_table = Array.new(256) do |i|
+              crc = i
+              8.times do
+                if crc.anybits?(1)
+                  crc = (crc >> 1) ^ 0xEDB88320
+                else
+                  crc >>= 1
+                end
+              end
+              crc & 0xFF
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/implementations/seven_zip/lzma/range_decoder.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Ported from 7-Zip SDK C/LzmaDec.c
+# Direct port of the LZMA SDK range decoder for byte-for-byte compatibility
+# with 7-Zip archives.
+
+require_relative "../../../algorithms/lzma/constants"
+require_relative "../../../algorithms/lzma/bit_model"
+
+module Omnizip
+  module Implementations
+    module SevenZip
+      module LZMA
+        # Range decoder for 7-Zip SDK LZMA decompression
+        #
+        # This is a direct port of 7-Zip SDK's range decoder implementation
+        # from LzmaDec.c for guaranteed compatibility with 7-Zip archives.
+        #
+        # Reference: /Users/mulgogi/src/external/7-Zip/C/LzmaDec.c
+        class RangeDecoder
+          include Omnizip::Algorithms::LZMA::Constants
+
+          attr_reader :code
+
+          # Initialize the range decoder
+          #
+          # @param input_stream [IO] The input stream of encoded bytes
+          def initialize(input_stream)
+            @stream = input_stream
+            @range = 0xFFFFFFFF
+            @code = 0
+            init_decoder
+          end
+
+          # Decode a single bit using a probability model
+          #
+          # Ported from 7-Zip SDK IF_BIT_0/UPDATE_0/UPDATE_1 macros
+          # (LzmaDec.c lines 22-26)
+          #
+          # SDK pattern:
+          #   #define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound)
+          #   #define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits));
+          #   #define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits));
+          #
+          # @param model [BitModel] The probability model for this bit
+          # @return [Integer] The decoded bit value (0 or 1)
+          def decode_bit(model)
+            prob = model.probability
+
+            # NORMALIZE (SDK pattern: normalize BEFORE decoding)
+            normalize
+
+            # Calculate bound
+            bound = (@range >> 11) * prob
+
+            if @code < bound
+              # UPDATE_0: bit is 0
+              @range = bound & 0xFFFFFFFF
+              new_prob = prob + ((BIT_MODEL_TOTAL - prob) >> MOVE_BITS)
+              model.instance_variable_set(:@probability, new_prob)
+              0
+            else
+              # UPDATE_1: bit is 1
+              @range = (@range - bound) & 0xFFFFFFFF
+              @code = (@code - bound) & 0xFFFFFFFF
+              new_prob = prob - (prob >> MOVE_BITS)
+              model.instance_variable_set(:@probability, new_prob)
+              1
+            end
+          end
+
+          # Decode bits directly without using probability model
+          #
+          # @param num_bits [Integer] Number of bits to decode
+          # @return [Integer] The decoded value
+          def decode_direct_bits(num_bits)
+            result = 0
+            num_bits.times do
+              normalize
+              @range >>= 1
+              @range &= 0xFFFFFFFF
+              @code = (@code - @range) & 0xFFFFFFFF
+              bit = (@code >> 31) & 1
+              @code = (@code + (@range & (0 - bit))) & 0xFFFFFFFF
+              result = (result << 1) | bit
+            end
+            result
+          end
+
+          # Decode bits directly with a base value
+          #
+          # Used by distance decoder for slots 14+ where we need to
+          # build on a base value (2 or 3) iteratively.
+          #
+          # @param num_bits [Integer] Number of bits to decode
+          # @param base [Integer] Base value to start from
+          # @return [Integer] The decoded value
+          def decode_direct_bits_with_base(num_bits, base)
+            result = base
+            num_bits.times do
+              result = (result << 1) + 1
+              normalize
+              @range >>= 1
+              @range &= 0xFFFFFFFF
+
+              # Check if bit is 1
+              bit = @code >= @range ? 1 : 0
+
+              if bit == 1
+                @code = (@code - @range) & 0xFFFFFFFF
+              else
+                result -= 1
+              end
+            end
+            result
+          end
+
+          # Update the input stream (for LZMA2 multi-chunk streams)
+          #
+          # @param new_stream [IO] New input stream
+          # @return [void]
+          def update_stream(new_stream)
+            @stream = new_stream
+          end
+
+          # Reset the decoder state (for LZMA2 chunks)
+          #
+          # @return [void]
+          def reset
+            @range = 0xFFFFFFFF
+            @code = 0
+            # Read initial 5 bytes for code
+            5.times { @code = ((@code << 8) | read_byte) & 0xFFFFFFFF }
+          end
+
+          private
+
+          # Initialize the decoder by reading the first 5 bytes
+          #
+          # @return [void]
+          def init_decoder
+            # Read first byte (should be 0 for valid LZMA stream)
+            first = read_byte
+            raise "Invalid LZMA stream: first byte not 0" unless first.zero?
+
+            # Read 4 bytes for initial code value
+            @code = 0
+            4.times { @code = ((@code << 8) | read_byte) & 0xFFFFFFFF }
+          end
+
+          # Normalize the range when it becomes too small
+          #
+          # Ported from 7-Zip SDK NORMALIZE macro (LzmaDec.c line 22):
+          #   #define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); }
+          #
+          # @return [void]
+          def normalize
+            while @range < TOP
+              @range = (@range << 8) & 0xFFFFFFFF
+              @code = ((@code << 8) | read_byte) & 0xFFFFFFFF
+            end
+          end
+
+          # Read a single byte from the input stream
+          #
+          # @return [Integer] The byte value (0-255)
+          def read_byte
+            byte = @stream.getbyte
+            if byte.nil?
+              raise Omnizip::DecompressionError,
+                    "LZMA compressed data exhausted prematurely"
+            end
+
+            byte
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/implementations/seven_zip/lzma/range_encoder.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Ported from 7-Zip SDK C/LzmaEnc.c
+# Direct port of the LZMA SDK range encoder for byte-for-byte compatibility
+# with 7-Zip archives.
+
+require_relative "../../../algorithms/lzma/range_coder"
+require_relative "../../../algorithms/lzma/constants"
+
+module Omnizip
+  module Implementations
+    module SevenZip
+      module LZMA
+        # Range encoder for 7-Zip SDK LZMA compression
+        #
+        # This is a direct port of 7-Zip SDK's range encoder implementation
+        # from LzmaEnc.c for guaranteed compatibility with 7-Zip archives.
+        #
+        # KEY DIFFERENCE from XZ Utils:
+        # - 7-Zip SDK normalizes AFTER encoding each bit
+        # - XZ Utils normalizes BEFORE encoding each bit
+        #
+        # This difference produces different output bytes, so we need
+        # separate implementations for 7-Zip and XZ Utils compatibility.
+        #
+        # Reference: /Users/mulgogi/src/external/7-Zip/C/LzmaEnc.c lines 730-784
+        class RangeEncoder
+          include Omnizip::Algorithms::LZMA::Constants
+
+          # Initialize the range encoder
+          #
+          # @param output_stream [IO] The output stream for encoded bytes
+          def initialize(output_stream)
+            @stream = output_stream
+            @low = 0
+            @range = 0xFFFFFFFF # Full 32-bit range
+            @cache = 0
+            @cache_size = 1 # SDK initializes to 1
+            @pre_flush_pos = 0
+          end
+
+          # Encode a single bit using a probability model
+          #
+          # Ported from 7-Zip SDK RC_BIT() macro (LzmaEnc.c lines 750-765)
+          # The key difference is that normalization happens AFTER encoding.
+          #
+          # SDK macro:
+          #   #define RC_BIT(p, prob, bit) { \
+          #     RC_BIT_PRE(p, prob) \
+          #     if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \
+          #     else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \
+          #     *(prob) = (CLzmaProb)ttt; \
+          #     RC_NORM(p) \
+          #   }
+          #
+          # @param model [BitModel] The probability model for this bit
+          # @param bit [Integer] The bit value (0 or 1)
+          # @return [void]
+          def encode_bit(model, bit)
+            prob = model.probability
+
+            # RC_BIT_PRE: Calculate newBound = (range >> kNumBitModelTotalBits) * prob
+            new_bound = (@range >> 11) * prob
+
+            new_prob = if bit.zero?
+                         # RC_BIT_0: shrink range to lower portion
+                         @range = new_bound & 0xFFFFFFFF
+                         # Update probability: ttt += (kBitModelTotal - ttt) >> kNumMoveBits
+                         prob + ((BIT_MODEL_TOTAL - prob) >> MOVE_BITS)
+                       else
+                         # RC_BIT_1: add bound to low, shrink range to upper portion
+                         @low = (@low + new_bound) & 0xFFFFFFFFFFFFFFFF
+                         @range = (@range - new_bound) & 0xFFFFFFFF
+                         # Update probability: ttt -= ttt >> kNumMoveBits
+                         prob - (prob >> MOVE_BITS)
+                       end
+            model.instance_variable_set(:@probability, new_prob)
+
+            # RC_NORM: Normalize AFTER encoding (key SDK difference!)
+            normalize
+          end
+
+          # Encode bits directly without using probability model
+          #
+          # Used for encoding values with uniform distribution (e.g., distance
+          # high bits).
+          #
+          # @param value [Integer] The value to encode
+          # @param num_bits [Integer] Number of bits to encode
+          # @return [void]
+          def encode_direct_bits(value, num_bits)
+            num_bits.times do |i|
+              @range >>= 1
+              @range &= 0xFFFFFFFF
+              bit = (value >> (num_bits - 1 - i)) & 1
+              @low = (@low + @range) & 0xFFFFFFFFFFFFFFFF if bit == 1
+              normalize
+            end
+          end
+
+          # Flush remaining bytes to output stream
+          #
+          # Writes the final bytes to complete the range coding stream.
+          #
+          # @return [void]
+          def flush
+            # Store position BEFORE flush for compatibility
+            @pre_flush_pos = @stream.pos
+
+            # Prevent further normalizations
+            @range = 0xFFFFFFFF
+
+            # Flush 5 bytes (matches SDK behavior)
+            5.times { shift_low }
+          end
+
+          # Return bytes needed for decoding
+          #
+          # @return [Integer] Number of bytes decoder will consume
+          def bytes_for_decode
+            @pre_flush_pos || @stream.pos
+          end
+
+          # Get current output position
+          #
+          # @return [Integer] Current position in output stream
+          def pos
+            @stream.pos
+          end
+
+          private
+
+          # Normalize the range when it becomes too small
+          #
+          # Ported from 7-Zip SDK RC_NORM macro (LzmaEnc.c line 730):
+          #   #define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); }
+          #
+          # @return [void]
+          def normalize
+            while @range < TOP
+              @range <<= 8
+              @range &= 0xFFFFFFFF
+              shift_low
+            end
+          end
+
+          # Shift the top byte of 'low' to output
+          #
+          # Ported from 7-Zip SDK RangeEnc_ShiftLow().
+          # Handles carry propagation through the cache mechanism.
+          #
+          # Reference: 7-Zip SDK C/LzmaEnc.c RangeEnc_ShiftLow
+          #
+          # @return [void]
+          def shift_low
+            low_32 = @low & 0xFFFFFFFF
+            carry = (@low >> 32) & 0xFF
+
+            if low_32 < 0xFF000000 || carry != 0
+              loop do
+                @stream.putc((@cache + carry) & 0xFF)
+                @cache = 0xFF
+                @cache_size -= 1
+                break if @cache_size.zero?
+              end
+
+              @cache = (low_32 >> 24) & 0xFF
+            end
+
+            @cache_size += 1
+            @low = (@low & 0x00FFFFFF) << 8
+          end
+        end
+      end
+    end
+  end
+end