omnizip 0.4.7-x86_64-linux

data/lib/omnizip/parity/chunked_block_processor.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Parity
+    # Chunked block processor for incremental Reed-Solomon recovery
+    #
+    # Implements correct RS decoding: x = A^-1 * b
+    # where b[i] = recovery[i] - sum(present[k] * base[present[k]]^exponent[i])
+    #
+    # Processes large blocks incrementally in memory-efficient chunks.
+    class ChunkedBlockProcessor
+      # Default chunk size (1MB)
+      DEFAULT_CHUNK_SIZE = 1024 * 1024
+
+      # @return [ReedSolomonMatrix] RS matrix with precomputed coefficients
+      attr_reader :matrix
+
+      # @return [Hash<Integer, String>] Present input blocks (index => data)
+      attr_reader :present_blocks
+
+      # @return [Hash<Integer, String>] Recovery blocks (exponent => data)
+      attr_reader :recovery_blocks
+
+      # @return [Array<Integer>] Missing block indices to recover
+      attr_reader :missing_indices
+
+      # @return [Integer] Block size in bytes
+      attr_reader :block_size
+
+      # @return [Integer] Chunk size for processing
+      attr_reader :chunk_size
+
+      # Initialize processor
+      #
+      # @param matrix [ReedSolomonMatrix] Precomputed RS matrix (A^-1)
+      # @param present_blocks [Hash<Integer, String>] Present data blocks
+      # @param recovery_blocks [Hash<Integer, String>] Recovery blocks (by exponent)
+      # @param missing_indices [Array<Integer>] Indices to recover
+      # @param block_size [Integer] Block size in bytes
+      # @param chunk_size [Integer] Chunk size for processing
+      def initialize(matrix, present_blocks, recovery_blocks, missing_indices,
+ block_size, chunk_size: DEFAULT_CHUNK_SIZE)
+        @matrix = matrix
+        @present_blocks = present_blocks
+        @recovery_blocks = recovery_blocks
+        @missing_indices = missing_indices.sort
+        @block_size = block_size
+        # Ensure chunk_size is even (we process 16-bit words)
+        requested_chunk = [chunk_size, block_size].min
+        @chunk_size = requested_chunk - (requested_chunk % 2)
+      end
+
+      # Process all blocks incrementally
+      #
+      # Implements: x = A^-1 * b
+      # where b = recovery - present_contributions
+      #
+      # @return [Hash<Integer, String>] Recovered blocks
+      def process_all
+        # Initialize output blocks (all zeros)
+        recovered = {}
+        missing_indices.each do |idx|
+          recovered[idx] = "\x00".b * block_size
+        end
+
+        # Process block chunk by chunk
+        block_offset = 0
+        while block_offset < block_size
+          current_chunk_size = [chunk_size, block_size - block_offset].min
+          process_chunk_at_offset(recovered, block_offset, current_chunk_size)
+          block_offset += current_chunk_size
+        end
+
+        recovered
+      end
+
+      private
+
+      # Process one chunk at given offset
+      #
+      # For each chunk, we:
+      # 1. Compute b_vector chunks from recovery blocks
+      # 2. Subtract present block contributions to get final b
+      # 3. Apply A^-1 matrix to b to get recovered chunks
+      #
+      # @param recovered [Hash<Integer, String>] Recovered blocks being built
+      # @param offset [Integer] Current offset within blocks
+      # @param length [Integer] Chunk length
+      def process_chunk_at_offset(recovered, offset, length)
+        # Step 1: Initialize b vector from recovery blocks
+        # b_vector[i] starts as recovery[i] (for each recovery exponent used)
+        b_vector = compute_b_vector_chunks(offset, length)
+
+        # Step 2: Apply A^-1 to b vector to get recovered chunks
+        # For each missing block j: recovered[j] = sum(A^-1[j,i] * b[i])
+        apply_inverse_matrix(recovered, b_vector, offset, length)
+      end
+
+      # Compute b vector chunks
+      #
+      # b[i] = recovery[i] - sum(present[k] * base[present[k]]^exponent[i])
+      #
+      # @param offset [Integer] Offset within blocks
+      # @param length [Integer] Chunk length
+      # @return [Array<String>] B vector chunks (one per recovery block used)
+      def compute_b_vector_chunks(offset, length)
+        b_chunks = []
+
+        # For each recovery block being used (must match matrix computation!)
+        matrix.used_recovery_exponents.each_with_index do |exponent, _exp_idx|
+          # Start with recovery block chunk
+          recovery_data = recovery_blocks[exponent]
+          b_chunk = recovery_data[offset, length].dup
+
+          # Subtract contributions from present blocks
+          present_blocks.each do |present_idx, present_data|
+            coefficient = matrix.present_contribution_coefficient(present_idx,
+                                                                  exponent)
+            next if coefficient.zero?
+
+            present_chunk = present_data[offset, length]
+
+            # Subtract: b -= present * coefficient
+            # In GF(2^16), subtraction is XOR, so: b ^= present * coefficient
+            subtract_contribution(b_chunk, present_chunk, coefficient, length)
+          end
+
+          b_chunks << b_chunk
+        end
+
+        b_chunks
+      end
+
+      # Subtract present block contribution from b chunk
+      #
+      # b_chunk ^= present_chunk * coefficient (GF subtraction is XOR)
+      #
+      # @param b_chunk [String] B vector chunk (modified in place)
+      # @param present_chunk [String] Present block chunk
+      # @param coefficient [Integer] Galois field coefficient
+      # @param length [Integer] Chunk length
+      def subtract_contribution(b_chunk, present_chunk, coefficient, length)
+        num_words = length / 2
+
+        num_words.times do |i|
+          word_offset = i * 2
+
+          # Read present word
+          present_word = present_chunk.getbyte(word_offset) |
+            (present_chunk.getbyte(word_offset + 1) << 8)
+
+          # Read current b word
+          b_word = b_chunk.getbyte(word_offset) |
+            (b_chunk.getbyte(word_offset + 1) << 8)
+
+          # Compute: b ^= present * coefficient
+          contribution = Galois16.multiply(present_word, coefficient)
+          result = Galois16.add(b_word, contribution) # add is XOR in GF(2^16)
+
+          # Write back
+          b_chunk.setbyte(word_offset, result & 0xFF)
+          b_chunk.setbyte(word_offset + 1, (result >> 8) & 0xFF)
+        end
+      end
+
+      # Apply inverse matrix to b vector
+      #
+      # For each missing block j:
+      #   recovered[j] += sum_i(A^-1[j,i] * b[i])
+      #
+      # @param recovered [Hash<Integer, String>] Recovered blocks being built
+      # @param b_vector [Array<String>] B vector chunks
+      # @param offset [Integer] Offset within blocks
+      # @param length [Integer] Chunk length
+      def apply_inverse_matrix(recovered, b_vector, offset, length)
+        # For each output (missing block)
+        missing_indices.each_with_index do |missing_idx, output_idx|
+          output_block = recovered[missing_idx]
+
+          # For each b vector element (recovery block)
+          b_vector.each_with_index do |b_chunk, recovery_idx|
+            # Get coefficient from A^-1
+            coefficient = matrix.coefficient(output_idx, recovery_idx)
+            next if coefficient.zero?
+
+            # CRITICAL: Accumulate chunk at the correct offset
+            matrix.process_chunk(coefficient, b_chunk, output_block, length,
+                                 output_offset: offset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/parity/galois16.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Parity
+    # Pure implementation of Galois Field GF(2^16) arithmetic
+    # Uses generator polynomial 0x1100B (69643) as per PAR2 specification
+    #
+    # This is pure algorithm code with no I/O dependencies.
+    # All operations are exact (no floating point) and work in GF(2^16).
+    class Galois16
+      BITS = 16
+      GENERATOR = 0x1100B # 69643 in decimal
+      FIELD_SIZE = 1 << BITS # 65536
+      LIMIT = FIELD_SIZE - 1 # 65535
+
+      # Lookup tables for efficient operations
+      @log_table = nil
+      @antilog_table = nil
+
+      class << self
+        # Build log and antilog tables
+        # This is called once when the class is loaded
+        def build_tables
+          return if @log_table && @antilog_table
+
+          @log_table = Array.new(FIELD_SIZE)
+          @antilog_table = Array.new(FIELD_SIZE)
+
+          b = 1
+          LIMIT.times do |l|
+            @log_table[b] = l
+            @antilog_table[l] = b
+
+            b <<= 1
+            b ^= GENERATOR if b.anybits?(FIELD_SIZE)
+          end
+
+          # Special cases for zero
+          @log_table[0] = LIMIT
+          @antilog_table[LIMIT] = 0
+        end
+
+        # Get log value (discrete logarithm)
+        def log(value)
+          @log_table[value & 0xFFFF]
+        end
+
+        # Get antilog value (inverse of log)
+        def antilog(value)
+          @antilog_table[value % FIELD_SIZE]
+        end
+
+        # Addition in GF(2^16) - same as subtraction (XOR)
+        def add(a, b)
+          (a ^ b) & 0xFFFF
+        end
+
+        # Subtraction in GF(2^16) - same as addition (XOR)
+        alias subtract add
+
+        # Multiplication in GF(2^16)
+        def multiply(a, b)
+          a &= 0xFFFF
+          b &= 0xFFFF
+
+          return 0 if a.zero? || b.zero?
+
+          sum = @log_table[a] + @log_table[b]
+          sum -= LIMIT if sum >= LIMIT
+          @antilog_table[sum]
+        end
+
+        # Division in GF(2^16)
+        def divide(a, b)
+          a &= 0xFFFF
+          b &= 0xFFFF
+
+          return 0 if a.zero?
+          raise ArgumentError, "Division by zero in GF(2^16)" if b.zero?
+
+          diff = @log_table[a] - @log_table[b]
+          diff += LIMIT if diff.negative?
+          @antilog_table[diff]
+        end
+
+        # Power in GF(2^16): compute a^n
+        def power(a, n)
+          a &= 0xFFFF
+
+          return 1 if n.zero?
+          return 0 if a.zero?
+
+          product = @log_table[a] * n
+
+          # Reduce modulo LIMIT using the identity:
+          # product mod LIMIT = (product >> BITS) + (product & LIMIT)
+          product = (product >> BITS) + (product & LIMIT)
+          product -= LIMIT if product >= LIMIT
+
+          @antilog_table[product]
+        end
+
+        # Compute GCD using Euclidean algorithm
+        def gcd(a, b)
+          while a != 0 && b != 0
+            if a > b
+              a %= b
+            else
+              b %= a
+            end
+          end
+          a + b
+        end
+
+        # Select base values for Reed-Solomon matrix
+        # Par2cmdline uses sequential logbases: base[i] = antilog[i]
+        # base[0] = antilog[0] = 1, base[1] = antilog[1] = 2, etc.
+        def select_bases(count)
+          raise ArgumentError, "Too many bases requested" if count >= LIMIT
+
+          bases = []
+
+          count.times do |i|
+            # Par2cmdline uses logbase = i (NOT i+1!)
+            # This gives: base[0]=1, base[1]=2, base[2]=4, etc.
+            logbase = i
+
+            if logbase >= LIMIT
+              raise ArgumentError,
+                    "Too many input blocks for Reed Solomon matrix"
+            end
+
+            # Convert log to actual value
+            bases << @antilog_table[logbase]
+          end
+
+          bases
+        end
+      end
+
+      # Build tables when class is loaded
+      build_tables
+    end
+  end
+end

data/lib/omnizip/parity/models/creator_packet.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Parity
+    module Models
+      # PAR2 Creator packet
+      #
+      # Contains information about the tool that created the PAR2 file.
+      # This packet is optional but recommended for identification.
+      #
+      # Body structure:
+      # - creator_string (variable): Null-terminated string identifying the tool
+      class CreatorPacket < Packet
+        # Packet type identifier
+        PACKET_TYPE = "PAR 2.0\x00Creator\x00"
+
+        # Creator identification string (without null terminator)
+        attribute :creator_string, :string, default: -> { "Omnizip PAR2" }
+
+        # Initialize creator packet
+        #
+        # @param attributes [Hash] Packet attributes
+        def initialize(**attributes)
+          super
+          self.type = PACKET_TYPE
+        end
+
+        # Parse body data into attributes
+        #
+        # Body format:
+        # - creator_string: null-terminated string
+        def parse_body
+          return if body_data.nil? || body_data.empty?
+
+          # Read creator string (null-terminated)
+          self.creator_string = body_data.unpack1("Z*")
+        end
+
+        # Build body data from attributes
+        #
+        # Constructs binary body data with null terminator
+        #
+        # @return [String] Binary body data
+        def build_body
+          data = +""
+
+          # Write creator string (null-terminated)
+          data << creator_string
+          data << "\x00"
+
+          self.body_data = data
+        end
+
+        # Get creator tool name
+        #
+        # @return [String] Creator tool name
+        def tool_name
+          creator_string.split.first || creator_string
+        end
+
+        # Get creator version if available
+        #
+        # @return [String, nil] Version string or nil
+        def version
+          parts = creator_string.split
+          parts.size > 1 ? parts[1..].join(" ") : nil
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/parity/models/file_description_packet.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Parity
+    module Models
+      # PAR2 File Description packet
+      #
+      # Contains metadata about a protected file including hashes and filename.
+      #
+      # Body structure:
+      # - file_id (16 bytes): MD5 hash identifying the file
+      # - file_hash (16 bytes): MD5 hash of complete file content
+      # - file_hash_16k (16 bytes): MD5 hash of first 16KB
+      # - length (8 bytes, Q<): File size in bytes
+      # - filename (variable): Null-terminated filename, padded to multiple of 4
+      class FileDescriptionPacket < Packet
+        # Packet type identifier
+        PACKET_TYPE = "PAR 2.0\x00FileDesc"
+
+        # File identifier (16 bytes MD5)
+        attribute :file_id, :string
+
+        # Full file hash (16 bytes MD5)
+        attribute :file_hash, :string
+
+        # First 16KB hash (16 bytes MD5)
+        attribute :file_hash_16k, :string
+
+        # File length in bytes
+        attribute :length, :integer
+
+        # Filename (without null terminator or padding)
+        attribute :filename, :string
+
+        # Initialize file description packet
+        #
+        # @param attributes [Hash] Packet attributes
+        def initialize(**attributes)
+          super
+          self.type = PACKET_TYPE
+        end
+
+        # Parse body data into attributes
+        #
+        # Body format:
+        # - file_id: 16 bytes
+        # - file_hash: 16 bytes
+        # - file_hash_16k: 16 bytes
+        # - length: 8 bytes (Q<)
+        # - filename: null-terminated, padded to multiple of 4
+        def parse_body
+          return if body_data.nil? || body_data.empty?
+
+          # Validate minimum size (16+16+16+8 = 56 bytes minimum)
+          if body_data.bytesize < 56
+            warn "FileDescriptionPacket body too short: #{body_data.bytesize} bytes"
+            return
+          end
+
+          pos = 0
+
+          # Read file_id (16 bytes)
+          self.file_id = body_data[pos, 16]
+          pos += 16
+
+          # Read file_hash (16 bytes)
+          self.file_hash = body_data[pos, 16]
+          pos += 16
+
+          # Read file_hash_16k (16 bytes)
+          self.file_hash_16k = body_data[pos, 16]
+          pos += 16
+
+          # Read length (8 bytes, little-endian unsigned 64-bit)
+          length_data = body_data[pos, 8]
+          return if length_data.nil? || length_data.bytesize < 8
+
+          self.length = length_data.unpack1("Q<")
+          pos += 8
+
+          # Read filename (null-terminated, remaining bytes)
+          filename_data = body_data[pos..]
+          self.filename = filename_data&.unpack1("Z*") || ""
+        end
+
+        # Build body data from attributes
+        #
+        # Constructs binary body data with proper padding
+        #
+        # @return [String] Binary body data
+        def build_body
+          data = +""
+
+          # Validate file_id, file_hash, file_hash_16k are 16 bytes
+          [file_id, file_hash, file_hash_16k].each_with_index do |hash, idx|
+            field_name = %w[file_id file_hash file_hash_16k][idx]
+            if hash.bytesize != 16
+              raise ArgumentError,
+                    "#{field_name} must be 16 bytes, got #{hash.bytesize}"
+            end
+          end
+
+          # Write hashes
+          data << file_id
+          data << file_hash
+          data << file_hash_16k
+
+          # Write length (8 bytes, little-endian)
+          data << [length].pack("Q<")
+
+          # Write filename (null-terminated, padded to multiple of 4)
+          data << filename
+          data << "\x00"
+
+          # Add padding to make total length multiple of 4
+          padding = (4 - ((filename.bytesize + 1) % 4)) % 4
+          data << ("\x00" * padding) if padding.positive?
+
+          self.body_data = data
+        end
+
+        # Get basename of file
+        #
+        # @return [String] File basename
+        def basename
+          File.basename(filename)
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/parity/models/ifsc_packet.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Parity
+    module Models
+      # PAR2 IFSC (Input File Slice Checksum) packet
+      #
+      # Contains checksums for individual data blocks of a file.
+      # One IFSC packet contains checksums for ALL blocks of a file.
+      #
+      # Body structure:
+      # - file_id (16 bytes): File identifier
+      # - For each block:
+      #   - block_hash (16 bytes): MD5 hash of block data
+      #   - block_crc32 (4 bytes, L<): CRC32 of block data
+      class IfscPacket < Packet
+        # Packet type identifier
+        PACKET_TYPE = "PAR 2.0\x00IFSC\x00\x00\x00\x00"
+
+        # File identifier (16 bytes)
+        attribute :file_id, :string
+
+        # Array of block hashes (16 bytes MD5 each)
+        attribute :block_hashes, :string, collection: true, default: -> { [] }
+
+        # Array of block CRC32s (4 bytes each)
+        attribute :block_crc32s, :integer, collection: true, default: -> { [] }
+
+        # Initialize IFSC packet
+        #
+        # @param attributes [Hash] Packet attributes
+        def initialize(**attributes)
+          super
+          self.type = PACKET_TYPE
+          self.block_hashes = [] if block_hashes.nil?
+          self.block_crc32s = [] if block_crc32s.nil?
+        end
+
+        # Parse body data into attributes
+        #
+        # Body format:
+        # - file_id: 16 bytes
+        # - For each block:
+        #   - block_hash: 16 bytes
+        #   - block_crc32: 4 bytes (L<)
+        def parse_body
+          return if body_data.nil? || body_data.empty?
+          return if body_data.bytesize < 16
+
+          pos = 0
+
+          # Read file_id (16 bytes)
+          self.file_id = body_data[pos, 16]
+          pos += 16
+
+          # Read all blocks (each is 16 bytes hash + 4 bytes CRC = 20 bytes)
+          self.block_hashes = []
+          self.block_crc32s = []
+
+          while pos + 20 <= body_data.bytesize
+            # Read block_hash (16 bytes)
+            block_hash = body_data[pos, 16]
+            pos += 16
+
+            # Read block_crc32 (4 bytes, little-endian unsigned 32-bit)
+            block_crc32 = body_data[pos, 4].unpack1("L<")
+            pos += 4
+
+            block_hashes << block_hash
+            block_crc32s << block_crc32
+          end
+        end
+
+        # Build body data from attributes
+        #
+        # Constructs binary body data
+        #
+        # @return [String] Binary body data
+        def build_body
+          data = +""
+
+          # Validate file_id is 16 bytes
+          if file_id.bytesize != 16
+            raise ArgumentError,
+                  "file_id must be 16 bytes, got #{file_id.bytesize}"
+          end
+
+          # Write file_id
+          data << file_id
+
+          # Write all blocks
+          block_hashes.each_with_index do |block_hash, i|
+            # Validate block_hash is 16 bytes
+            if block_hash.bytesize != 16
+              raise ArgumentError,
+                    "block_hash must be 16 bytes, got #{block_hash.bytesize}"
+            end
+
+            # Write block_hash
+            data << block_hash
+
+            # Write block_crc32 (4 bytes, little-endian)
+            data << [block_crc32s[i]].pack("L<")
+          end
+
+          self.body_data = data
+        end
+      end
+    end
+  end
+end