omnizip 0.3.1

data/lib/omnizip/algorithms/deflate64/lz77_encoder.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require_relative "constants"
+
+module Omnizip
+  module Algorithms
+    class Deflate64
+      # LZ77 encoder with 64KB sliding window for Deflate64
+      class LZ77Encoder
+        include Constants
+
+        attr_reader :window_size
+
+        def initialize(window_size = DICTIONARY_SIZE)
+          @window_size = window_size
+          @window = []
+          @hash_table = {}
+          @position = 0
+        end
+
+        # Find matches in data and return array of literals and match tokens
+        #
+        # @param data [String] Input data to compress
+        # @return [Array<Hash>] Array of match tokens
+        def find_matches(data)
+          tokens = []
+          pos = 0
+
+          while pos < data.bytesize
+            match = find_longest_match(pos, data)
+
+            if match && match[:length] >= MIN_MATCH_LENGTH
+              tokens << {
+                type: :match,
+                length: match[:length],
+                distance: match[:distance],
+              }
+              pos += match[:length]
+            else
+              tokens << {
+                type: :literal,
+                value: data.getbyte(pos),
+              }
+              pos += 1
+            end
+
+            update_window(data, pos)
+          end
+
+          tokens
+        end
+
+        private
+
+        # Find longest match for current position
+        #
+        # @param pos [Integer] Current position in data
+        # @param data [String] Input data
+        # @return [Hash, nil] Match information or nil
+        def find_longest_match(pos, data)
+          return nil if pos + MIN_MATCH_LENGTH > data.bytesize
+
+          best_match = nil
+          best_length = MIN_MATCH_LENGTH - 1
+
+          # Calculate hash for current position
+          hash = calculate_hash(data, pos)
+          candidates = @hash_table[hash] || []
+
+          # Search through candidate matches
+          candidates.reverse.take(MAX_CHAIN_LENGTH).each do |candidate_pos|
+            distance = pos - candidate_pos
+            break if distance > MAX_DISTANCE
+
+            # Find match length
+            length = match_length(data, pos, candidate_pos)
+
+            if length > best_length
+              best_length = length
+              best_match = {
+                length: length,
+                distance: distance,
+              }
+
+              break if length >= NICE_MATCH
+            end
+          end
+
+          # Add current position to hash table
+          @hash_table[hash] ||= []
+          @hash_table[hash] << pos
+
+          best_match
+        end
+
+        # Calculate match length between two positions
+        #
+        # @param data [String] Input data
+        # @param pos1 [Integer] First position
+        # @param pos2 [Integer] Second position
+        # @return [Integer] Match length
+        def match_length(data, pos1, pos2)
+          max_length = [MAX_MATCH_LENGTH, data.bytesize - pos1].min
+          length = 0
+
+          while length < max_length &&
+              data.getbyte(pos1 + length) == data.getbyte(pos2 + length)
+            length += 1
+          end
+
+          length
+        end
+
+        # Calculate hash value for position
+        #
+        # @param data [String] Input data
+        # @param pos [Integer] Position to hash
+        # @return [Integer] Hash value
+        def calculate_hash(data, pos)
+          return 0 if pos + MIN_MATCH_LENGTH > data.bytesize
+
+          hash = 0
+          MIN_MATCH_LENGTH.times do |i|
+            hash = ((hash << HASH_SHIFT) ^
+              data.getbyte(pos + i)) & (HASH_SIZE - 1)
+          end
+          hash
+        end
+
+        # Update sliding window
+        #
+        # @param data [String] Input data
+        # @param pos [Integer] Current position
+        def update_window(data, pos)
+          @window << data.getbyte(pos - 1) if pos.positive?
+          @window.shift if @window.size > @window_size
+          @position = pos
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/deflate64.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require_relative "../algorithm"
+require "zlib"
+
+module Omnizip
+  module Algorithms
+    # Deflate64 (Enhanced Deflate) compression algorithm
+    #
+    # Extends standard Deflate with:
+    # - 64KB sliding window (vs 32KB)
+    # - Better compression for large files
+    # - ZIP compression method 9
+    #
+    # NOTE: This is a simplified implementation that uses standard
+    # Deflate internally, as true Deflate64 requires complex
+    # bit-level manipulation that is better handled by libraries
+    # specifically designed for it.
+    class Deflate64 < Algorithm
+      # Constants
+      DICTIONARY_SIZE = 65_536 # 64KB window
+
+      # Algorithm metadata
+      def self.metadata
+        {
+          name: "Deflate64",
+          type: :compression,
+          streaming_supported: true,
+          dictionary_size: DICTIONARY_SIZE,
+          compression_method: 9,
+          description: "Enhanced Deflate with 64KB window",
+        }
+      end
+
+      # Compress input stream to output stream
+      #
+      # @param input [IO] Input stream
+      # @param output [IO] Output stream
+      # @param options [Hash] Compression options
+      # @option options [Integer] :level Compression level (1-9)
+      def compress(input, output, options = {})
+        level = options[:level] || Zlib::DEFAULT_COMPRESSION
+
+        data = input.read
+        return if data.nil? || data.empty?
+
+        # Use Zlib::Deflate with maximum window size
+        deflater = Zlib::Deflate.new(
+          level,
+          Zlib::MAX_WBITS, # Maximum window size
+          Zlib::MAX_MEM_LEVEL,
+        )
+
+        compressed = deflater.deflate(data, Zlib::FINISH)
+        deflater.close
+
+        output.write(compressed)
+      end
+
+      # Decompress input stream to output stream
+      #
+      # @param input [IO] Input stream
+      # @param output [IO] Output stream
+      # @param options [Hash] Decompression options
+      def decompress(input, output, _options = {})
+        compressed = input.read
+        return if compressed.nil? || compressed.empty?
+
+        # Set output to binary mode if it's a StringIO
+        output.set_encoding(Encoding::BINARY) if output.respond_to?(:set_encoding)
+        output.binmode if output.respond_to?(:binmode)
+
+        # Use Zlib::Inflate with maximum window size
+        inflater = Zlib::Inflate.new(Zlib::MAX_WBITS)
+        decompressed = inflater.inflate(compressed)
+        inflater.close
+
+        # Force binary encoding to match original data
+        decompressed.force_encoding(Encoding::BINARY)
+
+        output.write(decompressed)
+      end
+
+      # Check if streaming is supported
+      #
+      # @return [Boolean] Always true for Deflate64
+      def self.streaming_supported?
+        true
+      end
+
+      # Get dictionary size
+      #
+      # @return [Integer] 64KB
+      def self.dictionary_size
+        DICTIONARY_SIZE
+      end
+
+      # Get compression method ID for ZIP format
+      #
+      # @return [Integer] Method 9
+      def self.compression_method
+        9
+      end
+    end
+  end
+end
+
+# Register algorithm
+Omnizip::AlgorithmRegistry.register(:deflate64, Omnizip::Algorithms::Deflate64)

data/lib/omnizip/algorithms/lzma/bit_model.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # Adaptive probability model for range coding
+      #
+      # This class manages probability states for individual bits in the
+      # range coder. It uses adaptive arithmetic coding where probabilities
+      # are updated based on actual bit values encountered during encoding
+      # or decoding.
+      #
+      # Ported from XZ Utils range_encoder.c probability model implementation.
+      class BitModel
+        PROB_INIT = 1024  # Initial probability (0.5)
+        MOVE_BITS = 5     # Probability update speed
+        MAX_PROB = 1 << 11 # 4096
+        BIT_MODEL_TOTAL = 0x800 # XZ Utils RC_BIT_MODEL_TOTAL = 2048
+
+        attr_reader :probability
+
+        # Initialize a new bit probability model
+        #
+        # @param initial_prob [Integer] Initial probability value (default: PROB_INIT)
+        def initialize(initial_prob = PROB_INIT)
+          @probability = initial_prob
+        end
+
+        # Update the probability model based on an actual bit value
+        #
+        # This method implements the XZ Utils adaptive algorithm:
+        # - If bit is 0: probability increases (shifts toward encoding 0)
+        # - If bit is 1: probability decreases (shifts toward encoding 1)
+        #
+        # The update uses a shift operation (MOVE_BITS) to control the
+        # adaptation rate. Smaller MOVE_BITS means faster adaptation.
+        #
+        # XZ Utils formula (lzma/lzma_encoder.c:RC_BIT_*):
+        #   bit 0: prob += (RC_BIT_MODEL_TOTAL - prob) >> RC_MOVE_BITS
+        #   bit 1: prob -= prob >> RC_MOVE_BITS
+        # where RC_BIT_MODEL_TOTAL = 2048, RC_MOVE_BITS = 5
+        #
+        # @param bit [Integer] The actual bit value (0 or 1)
+        # @return [void]
+        def update(bit)
+          if bit.zero?
+            # XZ Utils formula: prob += (RC_BIT_MODEL_TOTAL - prob) >> RC_MOVE_BITS
+            @probability += ((BIT_MODEL_TOTAL - @probability) >> MOVE_BITS)
+          else
+            # XZ Utils formula: prob -= prob >> RC_MOVE_BITS
+            @probability -= (@probability >> MOVE_BITS)
+          end
+        end
+
+        # @deprecated Use {update} instead (same functionality, XZ Utils compatible)
+        def update!(bit)
+          update(bit)
+        end
+
+        # Reset the probability model to initial state
+        #
+        # @return [void]
+        def reset
+          @probability = PROB_INIT
+        end
+
+        # Get the probability of encoding a 0 bit
+        #
+        # @return [Integer] Probability value (0..MAX_PROB)
+        def prob_0
+          @probability
+        end
+
+        # Get the probability of encoding a 1 bit
+        #
+        # @return [Integer] Probability value (0..MAX_PROB)
+        def prob_1
+          MAX_PROB - @probability
+        end
+
+        # Create a copy of this bit model
+        #
+        # @return [BitModel] A new BitModel with the same probability
+        def dup
+          BitModel.new(@probability)
+        end
+
+        # For range coder: get probability scaled to 11 bits (XZ Utils compatibility)
+        #
+        # This method returns the probability value in the format expected
+        # by the range coder for encoding/decoding operations.
+        #
+        # @return [Integer] Probability value (0..MAX_PROB)
+        def to_range
+          @probability
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/constants.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class LZMA
+      # LZMA algorithm constants
+      #
+      # This module contains all constants used by the LZMA algorithm,
+      # including range coding parameters, probability models, and
+      # compression limits.
+      module Constants
+        # Range coder constants
+        # TOP: Threshold for range normalization (2^24)
+        TOP = 0x01000000
+
+        # BIT_MODEL_TOTAL: Total probability range for bit models (2^11)
+        BIT_MODEL_TOTAL = 0x800
+
+        # BIT_MODEL_MOVE_BITS: Number of bits to shift for prob updates
+        MOVE_BITS = 5
+
+        # INIT_PROBS: Initial probability value (0.5 probability)
+        INIT_PROBS = BIT_MODEL_TOTAL >> 1
+
+        # Number of bits used in direct bit encoding
+        NUM_DIRECT_BITS = 8
+
+        # LZMA state constants
+        # Number of position bits for literal context (lp)
+        NUM_LIT_POS_BITS_MAX = 4
+
+        # Number of literal context bits (lc)
+        NUM_LIT_CONTEXT_BITS_MAX = 8
+
+        # Number of position bits (pb)
+        NUM_POS_BITS_MAX = 4
+
+        # Number of LZMA states (from state machine)
+        NUM_STATES = 12
+
+        # Dictionary size limits
+        DICT_SIZE_MIN = 1 << 12  # 4KB
+        DICT_SIZE_MAX = 1 << 30  # 1GB
+
+        # Match length constants
+        MATCH_LEN_MIN = 2
+        MATCH_LEN_MAX = 273
+
+        # Number of distance slots
+        NUM_DIST_SLOTS = 64
+
+        # Position states
+        POS_STATES_MAX = 1 << NUM_POS_BITS_MAX
+
+        # Literal coder size
+        LIT_SIZE_MAX = (1 << (NUM_LIT_POS_BITS_MAX +
+                                NUM_LIT_CONTEXT_BITS_MAX))
+
+        # Number of length to position states
+        NUM_LEN_TO_POS_STATES = 4
+
+        # Compression levels
+        COMPRESSION_LEVEL_MIN = 0
+        COMPRESSION_LEVEL_MAX = 9
+        COMPRESSION_LEVEL_DEFAULT = 5
+
+        # End of stream marker
+        EOS_MARKER = true
+
+        # SDK-specific encoding constants
+        # Length encoding constants
+        NUM_LEN_LOW_BITS = 3
+        NUM_LEN_MID_BITS = 3
+        NUM_LEN_HIGH_BITS = 8
+        LEN_LOW_SYMBOLS = 1 << NUM_LEN_LOW_BITS
+        LEN_MID_SYMBOLS = 1 << NUM_LEN_MID_BITS
+        LEN_HIGH_SYMBOLS = 1 << NUM_LEN_HIGH_BITS
+
+        # Distance encoding constants
+        NUM_DIST_SLOT_BITS = 6
+        DIST_ALIGN_BITS = 4
+        DIST_ALIGN_SIZE = 1 << DIST_ALIGN_BITS
+        START_POS_MODEL_INDEX = 4
+        END_POS_MODEL_INDEX = 14
+        NUM_FULL_DISTANCES = 1 << (END_POS_MODEL_INDEX >> 1)
+
+        # Distance slot calculation helper
+        DIST_SLOT_FAST_LIMIT = 1 << (NUM_DIST_SLOT_BITS + 1)
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/decoder.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+require_relative "xz_utils_decoder"
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # LZMA Decoder - Factory for LZMA decompression implementations
+      #
+      # This class provides a unified interface for LZMA decoding, delegating
+      # to the XZ Utils implementation for full compatibility.
+      #
+      # The decoder reads a stream that consists of:
+      # - Property byte (lc, lp, pb parameters)
+      # - Dictionary size (4 bytes)
+      # - Uncompressed size (8 bytes)
+      # - Compressed data
+      class Decoder
+        attr_reader :dict_size, :lc, :lp, :pb, :uncompressed_size
+
+        # Initialize the decoder
+        #
+        # @param input [IO] Input stream of compressed data
+        # @param options [Hash] Decoding options
+        # @option options [Boolean] :raw_mode Skip header parsing for raw LZMA (for LZMA2)
+        # @option options [Integer] :dict_size Dictionary size for raw mode
+        def initialize(input, options = {})
+          # Use XZ Utils LZMA decoder (full XZ Utils compatibility)
+          @impl = XzUtilsDecoder.new(input, options)
+
+          # Expose header info for backward compatibility
+          @lc = @impl.lc
+          @lp = @impl.lp
+          @pb = @impl.pb
+          @dict_size = @impl.dict_size
+          @uncompressed_size = @impl.uncompressed_size
+        end
+
+        # Decode a compressed stream
+        #
+        # @param output [IO, nil] Optional output stream (if nil, returns String)
+        # @param preserve_dict [Boolean] Whether to preserve dictionary from previous decode
+        # @return [String, Integer] Decompressed data or bytes written
+        def decode_stream(output = nil, preserve_dict: false)
+          @impl.decode_stream(output, preserve_dict: preserve_dict)
+        end
+
+        # Reset the decoder state for reuse with new properties
+        #
+        # This method is used by LZMA2 decoder for multi-chunk streams.
+        #
+        # @param new_lc [Integer, nil] New lc value (if nil, keeps current)
+        # @param new_lp [Integer, nil] New lp value (if nil, keeps current)
+        # @param new_pb [Integer, nil] New pb value (if nil, keeps current)
+        # @param preserve_dict [Boolean] If true, preserve dictionary state (pos, dict_full)
+        # @return [void]
+        def reset(new_lc: nil, new_lp: nil, new_pb: nil, preserve_dict: false)
+          @impl.reset(new_lc: new_lc, new_lp: new_lp, new_pb: new_pb,
+                      preserve_dict: preserve_dict)
+
+          # Update cached properties
+          @lc = @impl.lc
+          @lp = @impl.lp
+          @pb = @impl.pb
+        end
+
+        # Reset only state machine and rep distances, preserve probability models
+        #
+        # This method is used by LZMA2 decoder for multi-chunk streams.
+        #
+        # @return [void]
+        def reset_state_only
+          @impl.reset_state_only
+        end
+
+        # Prepare state reset - called BEFORE setting new input
+        #
+        # This method is used by LZMA2 decoder for multi-chunk streams.
+        #
+        # @return [void]
+        def prepare_state_reset
+          @impl.prepare_state_reset
+        end
+
+        # Reset state machine only - preserves rep distances
+        #
+        # This method is used by LZMA2 decoder for multi-chunk streams
+        # where we want to reset the state machine but preserve rep distances
+        # from the previous chunk (control >= 0xA0 but < 0xC0).
+        #
+        # @return [void]
+        def reset_state_machine_only
+          @impl.reset_state_machine_only
+        end
+
+        # Finish state reset - called AFTER setting new input
+        #
+        # This method is used by LZMA2 decoder for multi-chunk streams.
+        #
+        # @return [void]
+        def finish_state_reset
+          @impl.finish_state_reset
+        end
+
+        # Set new input stream for chunked decoding
+        #
+        # This method is used by LZMA2 decoder for multi-chunk streams.
+        #
+        # @param new_input [IO] New input stream
+        # @return [void]
+        def set_input(new_input)
+          @impl.set_input(new_input)
+        end
+
+        # Set uncompressed size for chunked decoding
+        #
+        # This method is used by LZMA2 decoder for multi-chunk streams.
+        #
+        # @param size [Integer] Uncompressed size
+        # @param allow_eopm [Boolean] Whether to allow end-of-stream marker
+        # @return [void]
+        def set_uncompressed_size(size, allow_eopm: true)
+          @impl.set_uncompressed_size(size, allow_eopm: allow_eopm)
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/lzma/dictionary.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Algorithms
+    class LZMA < Algorithm
+      # Circular buffer dictionary for LZMA sliding window
+      # Ported from XZ Utils lzma_decoder.c
+      class Dictionary
+        attr_reader :size, :position, :buffer
+
+        def initialize(size)
+          @size = size
+          @buffer = String.new(encoding: Encoding::BINARY)
+          @position = 0
+        end
+
+        # Append bytes to dictionary
+        def append(data)
+          data.each_byte do |byte|
+            @buffer << byte
+            @position += 1
+
+            # Trim if exceeds size
+            if @buffer.bytesize > @size
+              excess = @buffer.bytesize - @size
+              @buffer = @buffer.byteslice(excess..-1)
+            end
+          end
+        end
+
+        # Read bytes from dictionary at a distance back
+        def read_bytes(distance, length)
+          raise "Invalid distance: #{distance}" if distance > @buffer.bytesize
+
+          result = String.new(encoding: Encoding::BINARY)
+          src_pos = @buffer.bytesize - distance
+
+          length.times do |i|
+            byte = @buffer[(src_pos + i) % @buffer.bytesize]
+            result << byte
+          end
+
+          result
+        end
+
+        # Get byte at distance back
+        def get_byte(distance)
+          raise "Invalid distance: #{distance}" if distance > @buffer.bytesize
+
+          @buffer.getbyte(@buffer.bytesize - distance)
+        end
+
+        # Reset dictionary
+        def reset!
+          @buffer.clear
+          @position = 0
+        end
+
+        # Clone dictionary
+        def clone
+          dict = Dictionary.new(@size)
+          dict.instance_variable_set(:@buffer, @buffer.dup)
+          dict.instance_variable_set(:@position, @position)
+          dict
+        end
+      end
+    end
+  end
+end