omnizip 0.4.7-arm-linux

data/lib/omnizip/filters/bcj2/decoder.rb

@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+module Omnizip
+  module Filters
+    # BCJ2 decoder - reconstructs original data from 4 streams.
+    #
+    # Matches the modern 7-Zip SDK Bcj2.c decode loop (v18.06+):
+    # - Pre-allocated output buffer (no dynamic growth)
+    # - Handles E8 (CALL), E9 (JMP), and JCC (0x0F 0x80-0x8F) opcodes
+    # - Probability model: index 0=JCC, 1=E9, 2+prev_byte=E8
+    # - Inline range decoder with cached locals
+    class Bcj2Decoder
+      include Bcj2Constants
+
+      attr_reader :ip
+
+      # Initialize decoder.
+      #
+      # @param streams [Bcj2StreamData] The 4 input streams
+      # @param position [Integer] Starting instruction pointer
+      def initialize(streams, position = 0)
+        @streams = streams
+        @ip = position
+        @range = 0
+        @code = 0
+        @probs = Array.new(NUM_PROBS, INITIAL_PROB)
+
+        # Stream positions
+        @main_pos = 0
+        @call_pos = 0
+        @jump_pos = 0
+        @rc_pos = 0
+      end
+
+      # Decode 4 streams back to original data.
+      #
+      # Matches modern 7-Zip SDK Bcj2Dec_Decode pattern:
+      # - Pre-allocated output buffer sized to expected output
+      # - Single C-optimized regex scan for E8/E9/JCC opcodes
+      # - Direct setbyte writes (no intermediate String allocations)
+      # - Opcode written to output before range-coded decision (matches SDK)
+      #
+      # @param expected_size [Integer, nil] Expected output size for pre-allocation
+      # @return [String] Decoded binary data
+      # @raise [Omnizip::DecompressionError] If output exceeds buffer capacity when expected_size is nil
+      def decode(expected_size = nil)
+        main_data = @streams.main
+        main_size = main_data.bytesize
+
+        # Pre-allocate output buffer to expected size (C++ SDK: dic is pre-allocated)
+        # When expected_size is unknown, use a conservative upper bound
+        # Each byte can produce at most 5 bytes (opcode + 4-byte address), but
+        # in practice BCJ2 never expands more than 2x due to rarity of convertible opcodes
+        out_capacity = expected_size || (main_size * 2)
+        result = ("\0" * out_capacity).b
+        result_pos = 0
+
+        # Helper to grow buffer if needed (only when expected_size is nil)
+        grow_buffer = lambda do |required|
+          return if result_pos + required <= result.bytesize
+
+          if expected_size
+            raise Omnizip::DecompressionError,
+                  "BCJ2 output size (#{result_pos + required}) exceeds expected size (#{expected_size})"
+          end
+
+          # Dynamic growth when expected_size is unknown
+          new_capacity = [result.bytesize * 2, result_pos + required].max
+          new_buf = ("\0" * new_capacity).b
+          new_buf[0, result_pos] = result.byteslice(0, result_pos)
+          result = new_buf
+        end
+
+        # Cache stream references as locals (avoids repeated ivar lookups in hot loop)
+        call_data = @streams.call
+        jump_data = @streams.jump
+        rc_data = @streams.rc
+        call_size = call_data.bytesize
+        jump_size = jump_data.bytesize
+        rc_size = rc_data.bytesize
+        call_pos = @call_pos
+        jump_pos = @jump_pos
+        rc_pos = @rc_pos
+        probs = @probs
+        ip = @ip
+
+        # Initialize range decoder (read first 5 bytes from RC stream)
+        range = 0xFFFFFFFF
+        code = 0
+        5.times do
+          break if rc_pos >= rc_size
+
+          code = ((code << 8) | rc_data.getbyte(rc_pos)) & 0xFFFFFFFF
+          rc_pos += 1
+        end
+
+        # Regex to find E8, E9, or JCC (0x0F followed by 0x80-0x8F)
+        opcode_re = /[\xe8\xe9]|\x0f[\x80-\x8f]/n
+        main_pos = @main_pos
+        prev_byte = 0
+
+        while main_pos < main_size
+          # C-optimized scan for E8, E9, and JCC opcodes
+          next_pos = main_data.index(opcode_re, main_pos)
+
+          unless next_pos
+            # No more opcodes — bulk copy remaining bytes
+            chunk_len = main_size - main_pos
+            grow_buffer.call(chunk_len)
+            result[result_pos, chunk_len] =
+              main_data.byteslice(main_pos, chunk_len)
+            result_pos += chunk_len
+            ip += chunk_len
+            main_pos = main_size
+            break
+          end
+
+          # Bulk copy bytes before opcode
+          if next_pos > main_pos
+            chunk_len = next_pos - main_pos
+            grow_buffer.call(chunk_len)
+            result[result_pos, chunk_len] =
+              main_data.byteslice(main_pos, chunk_len)
+            result_pos += chunk_len
+            ip += chunk_len
+            main_pos = next_pos
+            # Track prev_byte from bulk copy (last byte before opcode in output)
+            prev_byte = main_data.getbyte(next_pos - 1)
+          end
+
+          byte = main_data.getbyte(main_pos)
+
+          if byte == 0x0F
+            # JCC: 0x0F followed by 0x80-0x8F conditional jump
+            # Ensure buffer has room for 2 opcode bytes + potential 4 address bytes
+            grow_buffer.call(6)
+            # Write 0x0F to output (SDK writes opcode bytes before range decision)
+            result.setbyte(result_pos, 0x0F)
+            result_pos += 1
+            ip += 1
+            main_pos += 1
+
+            # Read and write the 0x8x second opcode byte
+            byte = main_data.getbyte(main_pos)
+            result.setbyte(result_pos, byte)
+            result_pos += 1
+            ip += 1
+            main_pos += 1
+
+            prob_idx = 0 # JCC uses probability index 0
+            is_call = false
+          else
+            # E8 (CALL) or E9 (JMP) — write opcode before range decision
+            # Ensure buffer has room for 1 opcode byte + potential 4 address bytes
+            grow_buffer.call(5)
+            result.setbyte(result_pos, byte)
+            result_pos += 1
+            ip += 1
+            main_pos += 1
+
+            if byte == 0xE8
+              prob_idx = 2 + prev_byte  # E8 uses prev_byte context (256 models)
+              is_call = true
+            else
+              prob_idx = 1              # E9 uses fixed index 1
+              is_call = false
+            end
+          end
+
+          # Inline range decoder normalization
+          if range < TOP_VALUE
+            range = (range << 8) & 0xFFFFFFFF
+            rc_byte = rc_pos < rc_size ? rc_data.getbyte(rc_pos) : 0
+            code = ((code << 8) | rc_byte) & 0xFFFFFFFF
+            rc_pos += 1 if rc_pos < rc_size
+          end
+
+          prob = probs[prob_idx]
+          bound = (range >> BIT_MODEL_TOTAL_BITS) * prob
+
+          if code < bound
+            # Bit = 0: not convertible (opcode already written to output)
+            range = bound
+            probs[prob_idx] = prob + ((BIT_MODEL_TOTAL - prob) >> MOVE_BITS)
+            prev_byte = byte
+          else
+            # Bit = 1: convertible — read 4-byte address from call/jump stream
+            range -= bound
+            code -= bound
+            probs[prob_idx] = prob - (prob >> MOVE_BITS)
+
+            # Read 4 bytes big-endian from call stream (E8) or jump stream (E9/JCC)
+            if is_call
+              if call_pos + 4 <= call_size
+                addr = call_data.byteslice(call_pos, 4).unpack1("N")
+                call_pos += 4
+              else
+                addr = 0
+                4.times do |i|
+                  break if call_pos >= call_size
+
+                  addr |= call_data.getbyte(call_pos) << (24 - (i * 8))
+                  call_pos += 1
+                end
+              end
+            elsif jump_pos + 4 <= jump_size
+              addr = jump_data.byteslice(jump_pos, 4).unpack1("N")
+              jump_pos += 4
+            else
+              addr = 0
+              4.times do |i|
+                break if jump_pos >= jump_size
+
+                addr |= jump_data.getbyte(jump_pos) << (24 - (i * 8))
+                jump_pos += 1
+              end
+            end
+
+            # Convert absolute address to relative (ip already past opcode bytes)
+            addr = (addr - (ip + 4)) & 0xFFFFFFFF
+
+            # Write 4-byte little-endian address directly (no pack/Array alloc)
+            result.setbyte(result_pos, addr & 0xFF)
+            result.setbyte(result_pos + 1, (addr >> 8) & 0xFF)
+            result.setbyte(result_pos + 2, (addr >> 16) & 0xFF)
+            result.setbyte(result_pos + 3, (addr >> 24) & 0xFF)
+            result_pos += 4
+            ip += 4
+            prev_byte = (addr >> 24) & 0xFF
+          end
+        end
+
+        # Store final positions back
+        @main_pos = main_pos
+        @call_pos = call_pos
+        @jump_pos = jump_pos
+        @rc_pos = rc_pos
+        @range = range
+        @code = code
+        @ip = ip
+
+        # Return exact output (trim if pre-allocated too large)
+        result_pos < result.bytesize ? result.byteslice(0, result_pos) : result
+      end
+    end
+  end
+end

data/lib/omnizip/filters/bcj2/encoder.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+module Omnizip
+  module Filters
+    # BCJ2 encoder - splits data into 4 streams.
+    #
+    # NOTE: BCJ2 encoding is extremely complex and is not yet
+    # implemented. This is primarily needed for compression,
+    # while decoding (decompression) is the more common use case.
+    #
+    # BCJ2 encoding requires:
+    # - Range encoder implementation
+    # - Proper probability model management
+    # - Stream splitting logic
+    # - Address conversion to absolute
+    #
+    # This will be implemented in a future version.
+    class Bcj2Encoder
+      include Bcj2Constants
+
+      # Initialize encoder.
+      #
+      # @param data [String] Binary data to encode
+      # @param position [Integer] Starting instruction pointer
+      def initialize(data, position = 0)
+        @data = data
+        @position = position
+      end
+
+      # Encode data into 4 streams.
+      #
+      # @raise [NotImplementedError] BCJ2 encoding not yet impl
+      # @return [Bcj2StreamData] The 4 output streams
+      def encode
+        raise NotImplementedError,
+              "BCJ2 encoding is not yet implemented. " \
+              "BCJ2 is primarily used for decompression. " \
+              "For compression, use the simpler BCJ-x86 filter."
+      end
+    end
+  end
+end

data/lib/omnizip/filters/bcj2/stream_data.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+module Omnizip
+  module Filters
+    # Model class representing the 4 BCJ2 streams.
+    #
+    # BCJ2 splits data into:
+    # - Main stream: Non-convertible bytes
+    # - Call stream: CALL (E8) instruction addresses
+    # - Jump stream: JUMP (E9) instruction addresses
+    # - RC stream: Range coder probability data
+    class Bcj2StreamData
+      include Bcj2Constants
+
+      attr_accessor :main, :call, :jump, :rc
+
+      # Initialize empty streams.
+      #
+      # @return [Bcj2StreamData] New stream data instance
+      def initialize
+        @main = String.new(encoding: Encoding::BINARY)
+        @call = String.new(encoding: Encoding::BINARY)
+        @jump = String.new(encoding: Encoding::BINARY)
+        @rc = String.new(encoding: Encoding::BINARY)
+      end
+
+      # Get stream by index.
+      #
+      # @param index [Integer] Stream index (0-3)
+      # @return [String] Stream data
+      # @raise [ArgumentError] If index is invalid
+      def [](index)
+        case index
+        when STREAM_MAIN then @main
+        when STREAM_CALL then @call
+        when STREAM_JUMP then @jump
+        when STREAM_RC then @rc
+        else
+          raise ArgumentError, "Invalid stream index: #{index}"
+        end
+      end
+
+      # Set stream by index.
+      #
+      # @param index [Integer] Stream index (0-3)
+      # @param data [String] Stream data
+      # @return [String] The data that was set
+      # @raise [ArgumentError] If index is invalid
+      def []=(index, data)
+        case index
+        when STREAM_MAIN then @main = data
+        when STREAM_CALL then @call = data
+        when STREAM_JUMP then @jump = data
+        when STREAM_RC then @rc = data
+        else
+          raise ArgumentError, "Invalid stream index: #{index}"
+        end
+      end
+
+      # Get all streams as an array.
+      #
+      # @return [Array<String>] Array of 4 streams
+      def to_a
+        [@main, @call, @jump, @rc]
+      end
+
+      # Check if all streams are empty.
+      #
+      # @return [Boolean] True if all streams are empty
+      def empty?
+        @main.empty? && @call.empty? && @jump.empty? && @rc.empty?
+      end
+    end
+  end
+end

data/lib/omnizip/filters/bcj2.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+module Omnizip
+  module Filters
+    # BCJ2 filter for x86/x64 executables (4-stream variant).
+    #
+    # BCJ2 is an advanced version of BCJ that splits x86 executable code
+    # into 4 separate streams for maximum compression:
+    # - Main stream: Non-convertible bytes
+    # - Call stream: CALL (0xE8) instruction addresses
+    # - Jump stream: JUMP (0xE9) instruction addresses
+    # - RC stream: Range coder probability data
+    #
+    # This provides better compression than simple BCJ at the cost of
+    # increased complexity. BCJ2 requires special handling in archive
+    # formats - the 4 streams must be stored and retrieved separately.
+    #
+    # NOTE: Currently only decoding (decompression) is implemented.
+    # Encoding is extremely complex and will be added in a future version.
+    # For compression use cases, the simpler BCJ-x86 filter is recommended.
+    class Bcj2 < Omnizip::Filter
+      # Encode is not currently supported for BCJ2.
+      #
+      # @param _data [String] Binary data to encode
+      # @param _position [Integer] Current stream position
+      # @raise [NotImplementedError] BCJ2 encoding not yet implemented
+      # @return [String] Encoded binary data
+      def encode(_data, _position = 0)
+        raise NotImplementedError,
+              "BCJ2 encoding is not yet implemented. " \
+              "Use the simpler BCJ-x86 filter for compression, " \
+              "or wait for a future version with BCJ2 encoding support."
+      end
+
+      # Decode (postprocess) BCJ2 data after decompression.
+      #
+      # This method expects the 4 BCJ2 streams to be provided in a
+      # Bcj2StreamData object. In practice, this is called by the archive
+      # format reader (e.g., 7z reader) which handles splitting the
+      # compressed data into the 4 streams.
+      #
+      # @param data [Bcj2StreamData, String] The 4 BCJ2 streams or error
+      # @param position [Integer] Current stream position
+      # @raise [ArgumentError] If data is not a Bcj2StreamData object
+      # @return [String] Decoded binary data
+      def decode(data, position = 0)
+        unless data.is_a?(Bcj2StreamData)
+          raise ArgumentError,
+                "BCJ2 decode requires a Bcj2StreamData object with " \
+                "4 streams. This is typically handled by the archive " \
+                "format reader."
+        end
+
+        decoder = Bcj2Decoder.new(data, position)
+        decoder.decode
+      end
+
+      class << self
+        # Get metadata about this filter.
+        #
+        # @return [Hash] Filter metadata
+        def metadata
+          {
+            name: "BCJ2",
+            description: "Advanced 4-stream Branch/Call/Jump converter " \
+                         "for x86/x64 executables",
+            architecture: "x86/x64",
+            streams: 4,
+            complexity: "high",
+            compression_quality: "maximum",
+            limitations: "Encoding not yet implemented",
+          }
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/filters/bcj_arm.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+module Omnizip
+  module Filters
+    # BCJ filter for 32-bit ARM executables.
+    #
+    # This filter preprocesses ARM machine code by converting relative
+    # addresses in BL (Branch and Link - 0xEB) instructions to absolute
+    # addresses. ARM uses 4-byte aligned instructions with little-endian
+    # encoding.
+    #
+    # The filter improves compression by making branch targets
+    # position-independent. The offset in ARM BL instructions is stored
+    # as word offset (divided by 4), and is relative to PC+8.
+    class BcjArm < Omnizip::Filter
+      # ARM BL (Branch and Link) opcode
+      OPCODE_BL = 0xEB
+
+      # Size of ARM instruction (4 bytes, little-endian)
+      INSTRUCTION_SIZE = 4
+
+      # Offset mask (24-bit offset in BL instruction)
+      OFFSET_MASK = 0x00FFFFFF
+
+      # Encode (preprocess) ARM executable data for compression.
+      #
+      # Scans for BL (0xEB) opcodes and converts relative word offsets
+      # to absolute word offsets. ARM branch offset is relative to PC+8.
+      #
+      # @param data [String] Binary executable data
+      # @param position [Integer] Current stream position
+      # @return [String] Encoded binary data
+      def encode(data, position = 0)
+        return data.dup if data.bytesize < INSTRUCTION_SIZE
+
+        result = data.b
+        size = data.bytesize & ~(INSTRUCTION_SIZE - 1)
+        i = 0
+        # PC starts at position + 4 (as per C implementation)
+        pc = position + 4
+
+        while i < size
+          # Advance PC to current instruction position
+          current_pc = pc + i
+
+          # Check if last byte is 0xEB (BL instruction)
+          if result.getbyte(i + 3) == OPCODE_BL
+            # Extract full 32-bit instruction value
+            instruction = extract_uint32_le(result, i)
+
+            # Calculate word offset from PC
+            word_offset = current_pc >> 2
+
+            # Add word offset to instruction value
+            instruction += word_offset
+
+            # Mask to 24-bit and combine with opcode
+            instruction = (instruction & OFFSET_MASK) | 0xEB000000
+
+            write_uint32_le(result, i, instruction)
+          end
+
+          i += INSTRUCTION_SIZE
+        end
+
+        result
+      end
+
+      # Decode (postprocess) ARM executable data after decompression.
+      #
+      # Reverses the encoding by converting absolute word offsets back to
+      # relative word offsets.
+      #
+      # @param data [String] Binary executable data
+      # @param position [Integer] Current stream position
+      # @return [String] Decoded binary data
+      def decode(data, position = 0)
+        return data.dup if data.bytesize < INSTRUCTION_SIZE
+
+        result = data.b
+        size = data.bytesize & ~(INSTRUCTION_SIZE - 1)
+        i = 0
+        # PC starts at position + 4 (as per C implementation)
+        pc = position + 4
+
+        while i < size
+          # Advance PC to current instruction position
+          current_pc = pc + i
+
+          # Check if last byte is 0xEB (BL instruction)
+          if result.getbyte(i + 3) == OPCODE_BL
+            # Extract full 32-bit instruction value
+            instruction = extract_uint32_le(result, i)
+
+            # Calculate word offset from PC
+            word_offset = current_pc >> 2
+
+            # Subtract word offset from instruction value
+            instruction -= word_offset
+
+            # Mask to 24-bit and combine with opcode
+            instruction = (instruction & OFFSET_MASK) | 0xEB000000
+
+            write_uint32_le(result, i, instruction)
+          end
+
+          i += INSTRUCTION_SIZE
+        end
+
+        result
+      end
+
+      class << self
+        # Get metadata about this filter.
+        #
+        # @return [Hash] Filter metadata
+        def metadata
+          {
+            name: "BCJ-ARM",
+            description: "Branch converter for 32-bit ARM executables",
+            architecture: "ARM (32-bit)",
+            alignment: 4,
+            endian: "little",
+          }
+        end
+      end
+
+      private
+
+      # Extract an unsigned 32-bit little-endian integer from data.
+      #
+      # @param data [String] Binary data
+      # @param offset [Integer] Starting position
+      # @return [Integer] Unsigned 32-bit integer
+      def extract_uint32_le(data, offset)
+        bytes = data.byteslice(offset, INSTRUCTION_SIZE).bytes
+        bytes[0] |
+          (bytes[1] << 8) |
+          (bytes[2] << 16) |
+          (bytes[3] << 24)
+      end
+
+      # Write an unsigned 32-bit little-endian integer to data.
+      #
+      # @param data [String] Binary data (modified in place)
+      # @param offset [Integer] Starting position
+      # @param value [Integer] 32-bit integer to write
+      # @return [void]
+      def write_uint32_le(data, offset, value)
+        value &= 0xFFFFFFFF
+        data.setbyte(offset, value & 0xFF)
+        data.setbyte(offset + 1, (value >> 8) & 0xFF)
+        data.setbyte(offset + 2, (value >> 16) & 0xFF)
+        data.setbyte(offset + 3, (value >> 24) & 0xFF)
+      end
+    end
+  end
+end