omnizip 0.3.1

data/lib/omnizip/filters/bcj2/constants.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+module Omnizip
+  module Filters
+    module Bcj2Constants
+      # Number of output streams
+      NUM_STREAMS = 4
+
+      # Stream indices
+      STREAM_MAIN = 0 # Main data stream (non-convertible bytes)
+      STREAM_CALL = 1 # CALL instruction addresses (E8)
+      STREAM_JUMP = 2 # JUMP instruction addresses (E9)
+      STREAM_RC = 3   # Range coder probability stream
+
+      # x86 opcodes
+      OPCODE_CALL = 0xE8 # CALL instruction
+      OPCODE_JUMP = 0xE9 # JUMP instruction
+
+      # Size of x86 address (4 bytes, little-endian)
+      ADDRESS_SIZE = 4
+
+      # Range coder constants
+      TOP_VALUE = 1 << 24            # Range normalization threshold
+      BIT_MODEL_TOTAL_BITS = 11      # Probability model bits
+      BIT_MODEL_TOTAL = 1 << BIT_MODEL_TOTAL_BITS
+      MOVE_BITS = 5                  # Probability update shift
+
+      # Number of probability models (2 + 256)
+      # - 2 for general cases (not E8/E9, or 0F8x pattern)
+      # - 256 for byte-specific models when processing E8
+      NUM_PROBS = 2 + 256
+
+      # Initial probability value (50%)
+      INITIAL_PROB = BIT_MODEL_TOTAL >> 1
+    end
+  end
+end

data/lib/omnizip/filters/bcj2/decoder.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+require_relative "constants"
+require_relative "stream_data"
+
+module Omnizip
+  module Filters
+    # BCJ2 decoder - reconstructs original data from 4 streams.
+    #
+    # Combines:
+    # - Main stream (non-convertible bytes)
+    # - Call stream (CALL/E8 addresses)
+    # - Jump stream (JUMP/E9 addresses)
+    # - RC stream (range coder probability data)
+    class Bcj2Decoder
+      include Bcj2Constants
+
+      attr_reader :ip
+
+      # Initialize decoder.
+      #
+      # @param streams [Bcj2StreamData] The 4 input streams
+      # @param position [Integer] Starting instruction pointer
+      def initialize(streams, position = 0)
+        @streams = streams
+        @ip = position
+        @range = 0
+        @code = 0
+        @probs = Array.new(NUM_PROBS, INITIAL_PROB)
+
+        # Stream positions
+        @main_pos = 0
+        @call_pos = 0
+        @jump_pos = 0
+        @rc_pos = 0
+      end
+
+      # Decode 4 streams back to original data.
+      #
+      # @return [String] Decoded binary data
+      def decode
+        result = String.new(encoding: Encoding::BINARY)
+        init_range_decoder
+
+        loop do
+          break if @main_pos >= @streams.main.bytesize
+
+          byte = @streams.main.getbyte(@main_pos)
+          @main_pos += 1
+
+          # Check for CALL (E8) or JUMP (E9) opcodes
+          if [OPCODE_CALL, OPCODE_JUMP].include?(byte)
+            # Use range decoder to determine if convertible
+            if read_bit(get_prob_index(byte))
+              # Convertible - read address from call/jump stream
+              addr = read_address(byte)
+              result << byte.chr(Encoding::BINARY)
+              result << encode_int32_le(addr)
+              @ip += 5
+            else
+              # Not convertible - just copy byte
+              result << byte.chr(Encoding::BINARY)
+              @ip += 1
+            end
+          else
+            # Regular byte - just copy
+            result << byte.chr(Encoding::BINARY)
+            @ip += 1
+          end
+        end
+
+        result
+      end
+
+      private
+
+      # Initialize range decoder by reading first 5 bytes from RC stream.
+      #
+      # @return [void]
+      def init_range_decoder
+        @range = 0xFFFFFFFF
+        @code = 0
+
+        5.times do
+          break if @rc_pos >= @streams.rc.bytesize
+
+          @code = (@code << 8) | @streams.rc.getbyte(@rc_pos)
+          @rc_pos += 1
+        end
+      end
+
+      # Read a single bit from range coder.
+      #
+      # @param prob_index [Integer] Probability model index
+      # @return [Boolean] Decoded bit (true = 1, false = 0)
+      def read_bit(prob_index) # rubocop:disable Naming/PredicateMethod
+        normalize_range
+
+        prob = @probs[prob_index]
+        bound = (@range >> BIT_MODEL_TOTAL_BITS) * prob
+
+        if @code < bound
+          # Bit is 0
+          @range = bound
+          @probs[prob_index] += (BIT_MODEL_TOTAL - prob) >> MOVE_BITS
+          false
+        else
+          # Bit is 1
+          @range -= bound
+          @code -= bound
+          @probs[prob_index] -= prob >> MOVE_BITS
+          true
+        end
+      end
+
+      # Normalize range decoder if needed.
+      #
+      # @return [void]
+      def normalize_range
+        while @range < TOP_VALUE
+          @range <<= 8
+          next_byte = if @rc_pos < @streams.rc.bytesize
+                        @streams.rc.getbyte(@rc_pos)
+                      else
+                        0
+                      end
+          @code = (@code << 8) | next_byte
+          @rc_pos += 1 if @rc_pos < @streams.rc.bytesize
+        end
+      end
+
+      # Get probability model index for a byte.
+      #
+      # @param byte [Integer] Byte value
+      # @return [Integer] Probability model index
+      def get_prob_index(byte)
+        # Use byte-specific model for E8, general model for E9
+        byte == OPCODE_CALL ? (2 + (@ip & 0xFF)) : 0
+      end
+
+      # Read 32-bit address from call or jump stream.
+      #
+      # @param opcode [Integer] Opcode (E8 or E9)
+      # @return [Integer] Converted address
+      def read_address(opcode)
+        stream_pos = opcode == OPCODE_CALL ? @call_pos : @jump_pos
+        stream = opcode == OPCODE_CALL ? @streams.call : @streams.jump
+
+        # Read 4 bytes (big-endian in stream, stored as absolute)
+        addr = 0
+        4.times do |i|
+          break if stream_pos >= stream.bytesize
+
+          addr |= stream.getbyte(stream_pos) << (24 - (i * 8))
+          stream_pos += 1
+        end
+
+        # Update stream position
+        if opcode == OPCODE_CALL
+          @call_pos = stream_pos
+        else
+          @jump_pos = stream_pos
+        end
+
+        # Convert back to relative
+        addr - (@ip + 5)
+      end
+
+      # Encode 32-bit integer as little-endian bytes.
+      #
+      # @param value [Integer] Value to encode
+      # @return [String] 4-byte string
+      def encode_int32_le(value)
+        unsigned = value & 0xFFFFFFFF
+        [
+          unsigned & 0xFF,
+          (unsigned >> 8) & 0xFF,
+          (unsigned >> 16) & 0xFF,
+          (unsigned >> 24) & 0xFF,
+        ].pack("C*")
+      end
+    end
+  end
+end

data/lib/omnizip/filters/bcj2/encoder.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+require_relative "constants"
+require_relative "stream_data"
+
+module Omnizip
+  module Filters
+    # BCJ2 encoder - splits data into 4 streams.
+    #
+    # NOTE: BCJ2 encoding is extremely complex and is not yet
+    # implemented. This is primarily needed for compression,
+    # while decoding (decompression) is the more common use case.
+    #
+    # BCJ2 encoding requires:
+    # - Range encoder implementation
+    # - Proper probability model management
+    # - Stream splitting logic
+    # - Address conversion to absolute
+    #
+    # This will be implemented in a future version.
+    class Bcj2Encoder
+      include Bcj2Constants
+
+      # Initialize encoder.
+      #
+      # @param data [String] Binary data to encode
+      # @param position [Integer] Starting instruction pointer
+      def initialize(data, position = 0)
+        @data = data
+        @position = position
+      end
+
+      # Encode data into 4 streams.
+      #
+      # @raise [NotImplementedError] BCJ2 encoding not yet impl
+      # @return [Bcj2StreamData] The 4 output streams
+      def encode
+        raise NotImplementedError,
+              "BCJ2 encoding is not yet implemented. " \
+              "BCJ2 is primarily used for decompression. " \
+              "For compression, use the simpler BCJ-x86 filter."
+      end
+    end
+  end
+end

data/lib/omnizip/filters/bcj2/stream_data.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+require_relative "constants"
+
+module Omnizip
+  module Filters
+    # Model class representing the 4 BCJ2 streams.
+    #
+    # BCJ2 splits data into:
+    # - Main stream: Non-convertible bytes
+    # - Call stream: CALL (E8) instruction addresses
+    # - Jump stream: JUMP (E9) instruction addresses
+    # - RC stream: Range coder probability data
+    class Bcj2StreamData
+      include Bcj2Constants
+
+      attr_accessor :main, :call, :jump, :rc
+
+      # Initialize empty streams.
+      #
+      # @return [Bcj2StreamData] New stream data instance
+      def initialize
+        @main = String.new(encoding: Encoding::BINARY)
+        @call = String.new(encoding: Encoding::BINARY)
+        @jump = String.new(encoding: Encoding::BINARY)
+        @rc = String.new(encoding: Encoding::BINARY)
+      end
+
+      # Get stream by index.
+      #
+      # @param index [Integer] Stream index (0-3)
+      # @return [String] Stream data
+      # @raise [ArgumentError] If index is invalid
+      def [](index)
+        case index
+        when STREAM_MAIN then @main
+        when STREAM_CALL then @call
+        when STREAM_JUMP then @jump
+        when STREAM_RC then @rc
+        else
+          raise ArgumentError, "Invalid stream index: #{index}"
+        end
+      end
+
+      # Set stream by index.
+      #
+      # @param index [Integer] Stream index (0-3)
+      # @param data [String] Stream data
+      # @return [String] The data that was set
+      # @raise [ArgumentError] If index is invalid
+      def []=(index, data)
+        case index
+        when STREAM_MAIN then @main = data
+        when STREAM_CALL then @call = data
+        when STREAM_JUMP then @jump = data
+        when STREAM_RC then @rc = data
+        else
+          raise ArgumentError, "Invalid stream index: #{index}"
+        end
+      end
+
+      # Get all streams as an array.
+      #
+      # @return [Array<String>] Array of 4 streams
+      def to_a
+        [@main, @call, @jump, @rc]
+      end
+
+      # Check if all streams are empty.
+      #
+      # @return [Boolean] True if all streams are empty
+      def empty?
+        @main.empty? && @call.empty? && @jump.empty? && @rc.empty?
+      end
+    end
+  end
+end

data/lib/omnizip/filters/bcj2.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+require_relative "filter_base"
+require_relative "bcj2/constants"
+require_relative "bcj2/stream_data"
+require_relative "bcj2/decoder"
+require_relative "bcj2/encoder"
+
+module Omnizip
+  module Filters
+    # BCJ2 filter for x86/x64 executables (4-stream variant).
+    #
+    # BCJ2 is an advanced version of BCJ that splits x86 executable code
+    # into 4 separate streams for maximum compression:
+    # - Main stream: Non-convertible bytes
+    # - Call stream: CALL (0xE8) instruction addresses
+    # - Jump stream: JUMP (0xE9) instruction addresses
+    # - RC stream: Range coder probability data
+    #
+    # This provides better compression than simple BCJ at the cost of
+    # increased complexity. BCJ2 requires special handling in archive
+    # formats - the 4 streams must be stored and retrieved separately.
+    #
+    # NOTE: Currently only decoding (decompression) is implemented.
+    # Encoding is extremely complex and will be added in a future version.
+    # For compression use cases, the simpler BCJ-x86 filter is recommended.
+    class Bcj2 < FilterBase
+      # Encode is not currently supported for BCJ2.
+      #
+      # @param _data [String] Binary data to encode
+      # @param _position [Integer] Current stream position
+      # @raise [NotImplementedError] BCJ2 encoding not yet implemented
+      # @return [String] Encoded binary data
+      def encode(_data, _position = 0)
+        raise NotImplementedError,
+              "BCJ2 encoding is not yet implemented. " \
+              "Use the simpler BCJ-x86 filter for compression, " \
+              "or wait for a future version with BCJ2 encoding support."
+      end
+
+      # Decode (postprocess) BCJ2 data after decompression.
+      #
+      # This method expects the 4 BCJ2 streams to be provided in a
+      # Bcj2StreamData object. In practice, this is called by the archive
+      # format reader (e.g., 7z reader) which handles splitting the
+      # compressed data into the 4 streams.
+      #
+      # @param data [Bcj2StreamData, String] The 4 BCJ2 streams or error
+      # @param position [Integer] Current stream position
+      # @raise [ArgumentError] If data is not a Bcj2StreamData object
+      # @return [String] Decoded binary data
+      def decode(data, position = 0)
+        unless data.is_a?(Bcj2StreamData)
+          raise ArgumentError,
+                "BCJ2 decode requires a Bcj2StreamData object with " \
+                "4 streams. This is typically handled by the archive " \
+                "format reader."
+        end
+
+        decoder = Bcj2Decoder.new(data, position)
+        decoder.decode
+      end
+
+      class << self
+        # Get metadata about this filter.
+        #
+        # @return [Hash] Filter metadata
+        def metadata
+          {
+            name: "BCJ2",
+            description: "Advanced 4-stream Branch/Call/Jump converter " \
+                         "for x86/x64 executables",
+            architecture: "x86/x64",
+            streams: 4,
+            complexity: "high",
+            compression_quality: "maximum",
+            limitations: "Encoding not yet implemented",
+          }
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/filters/bcj_arm.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+require_relative "filter_base"
+
+module Omnizip
+  module Filters
+    # BCJ filter for 32-bit ARM executables.
+    #
+    # This filter preprocesses ARM machine code by converting relative
+    # addresses in BL (Branch and Link - 0xEB) instructions to absolute
+    # addresses. ARM uses 4-byte aligned instructions with little-endian
+    # encoding.
+    #
+    # The filter improves compression by making branch targets
+    # position-independent. The offset in ARM BL instructions is stored
+    # as word offset (divided by 4), and is relative to PC+8.
+    class BcjArm < FilterBase
+      # ARM BL (Branch and Link) opcode
+      OPCODE_BL = 0xEB
+
+      # Size of ARM instruction (4 bytes, little-endian)
+      INSTRUCTION_SIZE = 4
+
+      # Offset mask (24-bit offset in BL instruction)
+      OFFSET_MASK = 0x00FFFFFF
+
+      # Encode (preprocess) ARM executable data for compression.
+      #
+      # Scans for BL (0xEB) opcodes and converts relative word offsets
+      # to absolute word offsets. ARM branch offset is relative to PC+8.
+      #
+      # @param data [String] Binary executable data
+      # @param position [Integer] Current stream position
+      # @return [String] Encoded binary data
+      def encode(data, position = 0)
+        return data.dup if data.bytesize < INSTRUCTION_SIZE
+
+        result = data.b
+        size = data.bytesize & ~(INSTRUCTION_SIZE - 1)
+        i = 0
+        # PC starts at position + 4 (as per C implementation)
+        pc = position + 4
+
+        while i < size
+          # Advance PC to current instruction position
+          current_pc = pc + i
+
+          # Check if last byte is 0xEB (BL instruction)
+          if result.getbyte(i + 3) == OPCODE_BL
+            # Extract full 32-bit instruction value
+            instruction = extract_uint32_le(result, i)
+
+            # Calculate word offset from PC
+            word_offset = current_pc >> 2
+
+            # Add word offset to instruction value
+            instruction += word_offset
+
+            # Mask to 24-bit and combine with opcode
+            instruction = (instruction & OFFSET_MASK) | 0xEB000000
+
+            write_uint32_le(result, i, instruction)
+          end
+
+          i += INSTRUCTION_SIZE
+        end
+
+        result
+      end
+
+      # Decode (postprocess) ARM executable data after decompression.
+      #
+      # Reverses the encoding by converting absolute word offsets back to
+      # relative word offsets.
+      #
+      # @param data [String] Binary executable data
+      # @param position [Integer] Current stream position
+      # @return [String] Decoded binary data
+      def decode(data, position = 0)
+        return data.dup if data.bytesize < INSTRUCTION_SIZE
+
+        result = data.b
+        size = data.bytesize & ~(INSTRUCTION_SIZE - 1)
+        i = 0
+        # PC starts at position + 4 (as per C implementation)
+        pc = position + 4
+
+        while i < size
+          # Advance PC to current instruction position
+          current_pc = pc + i
+
+          # Check if last byte is 0xEB (BL instruction)
+          if result.getbyte(i + 3) == OPCODE_BL
+            # Extract full 32-bit instruction value
+            instruction = extract_uint32_le(result, i)
+
+            # Calculate word offset from PC
+            word_offset = current_pc >> 2
+
+            # Subtract word offset from instruction value
+            instruction -= word_offset
+
+            # Mask to 24-bit and combine with opcode
+            instruction = (instruction & OFFSET_MASK) | 0xEB000000
+
+            write_uint32_le(result, i, instruction)
+          end
+
+          i += INSTRUCTION_SIZE
+        end
+
+        result
+      end
+
+      class << self
+        # Get metadata about this filter.
+        #
+        # @return [Hash] Filter metadata
+        def metadata
+          {
+            name: "BCJ-ARM",
+            description: "Branch converter for 32-bit ARM executables",
+            architecture: "ARM (32-bit)",
+            alignment: 4,
+            endian: "little",
+          }
+        end
+      end
+
+      private
+
+      # Extract an unsigned 32-bit little-endian integer from data.
+      #
+      # @param data [String] Binary data
+      # @param offset [Integer] Starting position
+      # @return [Integer] Unsigned 32-bit integer
+      def extract_uint32_le(data, offset)
+        bytes = data.byteslice(offset, INSTRUCTION_SIZE).bytes
+        bytes[0] |
+          (bytes[1] << 8) |
+          (bytes[2] << 16) |
+          (bytes[3] << 24)
+      end
+
+      # Write an unsigned 32-bit little-endian integer to data.
+      #
+      # @param data [String] Binary data (modified in place)
+      # @param offset [Integer] Starting position
+      # @param value [Integer] 32-bit integer to write
+      # @return [void]
+      def write_uint32_le(data, offset, value)
+        value &= 0xFFFFFFFF
+        data.setbyte(offset, value & 0xFF)
+        data.setbyte(offset + 1, (value >> 8) & 0xFF)
+        data.setbyte(offset + 2, (value >> 16) & 0xFF)
+        data.setbyte(offset + 3, (value >> 24) & 0xFF)
+      end
+    end
+  end
+end