omnizip 0.4.7-arm-linux

data/lib/omnizip/parity/par2_verifier.rb

@@ -0,0 +1,373 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Omnizip
+  module Parity
+    # PAR2 archive verifier
+    #
+    # Verifies file integrity using PAR2 recovery files and checks
+    # if damaged files can be repaired.
+    #
+    # Uses PacketRegistry and packet models for clean object-oriented
+    # architecture.
+    #
+    # @example Verify files
+    #   verifier = Par2Verifier.new('backup.par2')
+    #   result = verifier.verify
+    #   if result.repairable?
+    #     puts "#{result.damaged_blocks.size} blocks damaged, can repair"
+    #   end
+    class Par2Verifier
+      # Verification result
+      VerificationResult = Struct.new(
+        :all_ok,          # All files intact?
+        :damaged_files,   # Array of damaged file names
+        :damaged_blocks,  # Array of damaged block indices
+        :missing_files,   # Array of missing file names
+        :repairable,      # Can be repaired with available parity?
+        :total_blocks,    # Total number of blocks
+        :recovery_blocks, # Number of recovery blocks available
+        keyword_init: true,
+      ) do
+        # Check if all files are OK
+        #
+        # @return [Boolean] true if no damage detected
+        def all_ok?
+          damaged_files.empty? && missing_files.empty?
+        end
+
+        # Check if damage can be repaired
+        #
+        # @return [Boolean] true if repairable
+        def repairable?
+          repairable
+        end
+
+        # Get total damage count
+        #
+        # @return [Integer] Number of damaged/missing blocks
+        def damage_count
+          damaged_blocks.size + missing_files.size
+        end
+      end
+
+      # @return [String] Path to PAR2 index file
+      attr_reader :par2_file
+
+      # @return [Hash] Parsed PAR2 metadata
+      attr_reader :metadata
+
+      # @return [Array<Hash>] Parsed file list (one entry per managed file)
+      attr_reader :file_list
+
+      # @return [Array] Available recovery blocks
+      attr_reader :recovery_blocks
+
+      # Initialize verifier
+      #
+      # @param par2_file [String] Path to .par2 index file
+      # @raise [ArgumentError] if file doesn't exist
+      def initialize(par2_file)
+        if par2_file.nil?
+          raise ArgumentError,
+                "PAR2 file is nil — no index .par2 was produced for this set"
+        end
+
+        raise ArgumentError, "PAR2 file not found: #{par2_file}" unless
+          File.exist?(par2_file)
+
+        @par2_file = par2_file
+        @metadata = {}
+        @file_list = []
+        @recovery_blocks = []
+        @block_hashes = {} # Store IFSC block hashes for verification
+      end
+
+      # Verify files against PAR2 data
+      #
+      # @return [VerificationResult] Verification results
+      def verify
+        parse_par2_file
+
+        damaged_files = []
+        damaged_blocks = []
+        missing_files = []
+
+        # Track global block position as we iterate through files
+        global_block_idx = 0
+
+        # Check each file
+        @file_list.each do |file_info|
+          file_path = find_file_path(file_info[:filename])
+          num_blocks = (file_info[:size].to_f / @metadata[:block_size]).ceil
+
+          if file_path.nil?
+            missing_files << file_info[:filename]
+            global_block_idx += num_blocks
+            next
+          end
+
+          # Verify file integrity
+          damage = verify_file(file_path, file_info)
+          if damage[:damaged]
+            damaged_files << file_info[:filename]
+            # Convert file-relative indices to global indices
+            damage[:blocks].each do |file_relative_idx|
+              damaged_blocks << (global_block_idx + file_relative_idx)
+            end
+          end
+
+          global_block_idx += num_blocks
+        end
+
+        # Check if repairable
+        total_damage = damaged_blocks.size + (missing_files.size * avg_blocks_per_file)
+        repairable = total_damage <= @recovery_blocks.size
+
+        VerificationResult.new(
+          all_ok: damaged_files.empty? && missing_files.empty?,
+          damaged_files: damaged_files,
+          damaged_blocks: damaged_blocks,
+          missing_files: missing_files,
+          repairable: repairable,
+          total_blocks: calculate_total_blocks,
+          recovery_blocks: @recovery_blocks.size,
+        )
+      end
+
+      # Parse the PAR2 index file and populate +metadata+, +file_list+,
+      # and +recovery_blocks+. Idempotent — safe to call multiple times.
+      #
+      # @return [void]
+      def parse_par2_file
+        @metadata = {}
+        @file_list = []
+        @recovery_blocks = []
+        @block_hashes = {}
+
+        File.open(@par2_file, "rb") do |io|
+          while !io.eof?
+            packet = Models::PacketRegistry.read_packet(io)
+            break unless packet
+
+            process_packet_model(packet)
+          end
+        end
+
+        # CRITICAL: Sort file_list by position in Main packet file_ids
+        # The Main packet defines canonical file order for Reed-Solomon matrix
+        # Do NOT sort by file_id string - that breaks recovery!
+        if @metadata[:file_ids]
+          file_id_order = {}
+          @metadata[:file_ids].each_with_index do |fid, idx|
+            file_id_order[fid] = idx
+          end
+          @file_list.sort_by! { |f| file_id_order[f[:file_id]] || 999 }
+        end
+
+        # Load recovery blocks from volume files
+        load_recovery_volumes
+      end
+
+      # Process main packet model
+      #
+      # @param packet [Models::MainPacket] Main packet
+      def process_main_packet_model(packet)
+        @metadata[:block_size] = packet.block_size
+        @metadata[:file_ids] = packet.file_ids.dup
+      end
+
+      # Process file description packet model
+      #
+      # @param packet [Models::FileDescriptionPacket] File description packet
+      def process_file_description_packet_model(packet)
+        # Skip packets with incomplete data
+        return if packet.file_id.nil? || packet.filename.nil? || packet.filename.empty?
+        return if packet.file_hash.nil? || packet.length.nil?
+
+        @file_list << {
+          file_id: packet.file_id,
+          hash_full: packet.file_hash,
+          hash_16k: packet.file_hash_16k,
+          size: packet.length,
+          filename: packet.filename,
+        }
+      end
+
+      # Process IFSC packet model
+      #
+      # Each IFSC packet contains checksums for all blocks of a file
+      #
+      # @param packet [Models::IfscPacket] IFSC packet
+      def process_ifsc_packet_model(packet)
+        # Store block hashes keyed by file_id
+        # Each IFSC packet contains all hashes for one file
+        @block_hashes[packet.file_id] = packet.block_hashes
+      end
+
+      # Process recovery packet model
+      #
+      # @param packet [Models::RecoverySlicePacket] Recovery packet
+      def process_recovery_packet_model(packet)
+        @recovery_blocks << {
+          exponent: packet.exponent,
+          data: packet.recovery_data,
+        }
+      end
+
+      # Load recovery blocks from volume files
+      def load_recovery_volumes
+        base_name = File.basename(@par2_file, ".par2")
+        dir_name = File.dirname(@par2_file)
+
+        # Find all volume files
+        pattern = File.join(dir_name, "#{base_name}.vol*.par2")
+        volume_files = Dir.glob(pattern)
+
+        volume_files.each do |volume_file|
+          load_recovery_volume(volume_file)
+        end
+      end
+
+      # Load recovery blocks from single volume file
+      #
+      # @param volume_file [String] Path to volume file
+      def load_recovery_volume(volume_file)
+        File.open(volume_file, "rb") do |io|
+          while !io.eof?
+            packet = Models::PacketRegistry.read_packet(io)
+            break unless packet
+
+            # Only process recovery packets from volume files
+            process_packet_model(packet) if packet.is_a?(Models::RecoverySlicePacket)
+          end
+        end
+      end
+
+      # Find file path for filename
+      #
+      # @param filename [String] Filename from PAR2
+      # @return [String, nil] Full path or nil if not found
+      def find_file_path(filename)
+        # Look in same directory as PAR2 file
+        dir = File.dirname(@par2_file)
+        candidate = File.join(dir, filename)
+
+        return candidate if File.exist?(candidate)
+
+        # Look in current directory
+        return filename if File.exist?(filename)
+
+        nil
+      end
+
+      # Verify single file
+      #
+      # @param file_path [String] Path to file
+      # @param file_info [Hash] Expected file information
+      # @return [Hash] Damage information
+      def verify_file(file_path, file_info)
+        damaged_blocks = []
+        damaged = false
+
+        File.open(file_path, "rb") do |io|
+          # Quick check: file size
+          if io.size != file_info[:size]
+            damaged = true
+            # Still identify damaged blocks even with size mismatch
+            damaged_blocks = identify_damaged_blocks(io, file_info)
+            return { damaged: damaged, blocks: damaged_blocks,
+                     size_mismatch: true }
+          end
+
+          # Quick check: first 16KB hash
+          first_16k = io.read(16384) || ""
+          hash_16k = Digest::MD5.digest(first_16k)
+          if hash_16k == file_info[:hash_16k]
+            # Full check: complete file hash
+            io.rewind
+            hash_full = Digest::MD5.file(file_path).digest
+            if hash_full != file_info[:hash_full]
+              damaged = true
+              # Identify damaged blocks
+              damaged_blocks = identify_damaged_blocks(io, file_info)
+            end
+          else
+            damaged = true
+            # Identify damaged blocks when quick check fails
+            damaged_blocks = identify_damaged_blocks(io, file_info)
+          end
+        end
+
+        { damaged: damaged, blocks: damaged_blocks }
+      end
+
+      # Identify which blocks are damaged
+      #
+      # @param io [IO] File IO
+      # @param file_info [Hash] File information
+      # @return [Array<Integer>] Damaged block indices
+      def identify_damaged_blocks(io, file_info)
+        damaged = []
+        block_idx = 0
+        expected_hashes = @block_hashes[file_info[:file_id]] || []
+
+        io.rewind
+        while (data = io.read(@metadata[:block_size]))
+          # Pad last block
+          if data.bytesize < @metadata[:block_size]
+            data += "\x00" * (@metadata[:block_size] - data.bytesize)
+          end
+
+          # Check block hash against stored IFSC hash
+          block_hash = Digest::MD5.digest(data)
+          if block_idx < expected_hashes.size && block_hash != expected_hashes[block_idx]
+            damaged << block_idx
+          end
+
+          block_idx += 1
+        end
+
+        damaged
+      end
+
+      # Calculate average blocks per file
+      #
+      # @return [Integer] Average blocks per file
+      def avg_blocks_per_file
+        return 0 if @file_list.empty?
+
+        total_size = @file_list.sum { |f| f[:size] }
+        total_blocks = (total_size.to_f / @metadata[:block_size]).ceil
+        (total_blocks.to_f / @file_list.size).ceil
+      end
+
+      # Calculate total blocks
+      #
+      # @return [Integer] Total number of blocks
+      def calculate_total_blocks
+        @file_list.sum do |file_info|
+          (file_info[:size].to_f / @metadata[:block_size]).ceil
+        end
+      end
+
+      private
+
+      def process_packet_model(packet)
+        case packet
+        when Models::MainPacket
+          process_main_packet_model(packet)
+        when Models::FileDescriptionPacket
+          process_file_description_packet_model(packet)
+        when Models::IfscPacket
+          process_ifsc_packet_model(packet)
+        when Models::RecoverySlicePacket
+          process_recovery_packet_model(packet)
+        when Models::CreatorPacket
+          # Creator packets are informational only
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/parity/par2cmdline_algorithm.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Parity
+    # Par2cmdline-compatible Reed-Solomon algorithm
+    #
+    # This implements the EXACT algorithm from par2cmdline reedsolomon.cpp
+    # Lines 230-262 show the base value computation for encoding.
+    #
+    # Key algorithm from par2cmdline:
+    #   unsigned int logbase = 0;
+    #   for (unsigned int index=0; index<count; index++)
+    #   {
+    #     while (gcd(G::Limit, logbase) != 1)
+    #     {
+    #       logbase++;
+    #     }
+    #     G::ValueType base = G(logbase++).ALog();
+    #     database[index] = base;
+    #   }
+    #
+    # Then for matrix: coefficient = base[col] ^ exponent
+    #
+    module Par2cmdlineAlgorithm
+      # Compute base values exactly as par2cmdline does
+      #
+      # @param data_count [Integer] Number of data blocks
+      # @return [Array<Integer>] Base values for each data block
+      def self.compute_bases(data_count)
+        bases = []
+        logbase = 0
+        limit = 65535 # GF(2^16) limit
+
+        data_count.times do
+          # Find next logbase where gcd(65535, logbase) == 1
+          logbase += 1 while gcd(limit, logbase) != 1
+
+          raise ArgumentError, "Too many input blocks" if logbase >= limit
+
+          # Use antilog to convert logbase to base value
+          # This is the key: base = antilog[logbase]
+          base = Galois16.antilog(logbase)
+          bases << base
+          logbase += 1
+        end
+
+        bases
+      end
+
+      # Build encoding matrix using par2cmdline algorithm
+      #
+      # @param data_count [Integer] Number of data blocks
+      # @param recovery_count [Integer] Number of recovery blocks
+      # @return [Array<Array<Integer>>] Encoding matrix
+      def self.build_encoding_matrix(data_count, recovery_count)
+        bases = compute_bases(data_count)
+
+        matrix = Array.new(recovery_count) { Array.new(data_count) }
+
+        recovery_count.times do |exponent|
+          data_count.times do |col|
+            # Matrix coefficient = base[col] ^ exponent
+            matrix[exponent][col] = Galois16.power(bases[col], exponent)
+          end
+        end
+
+        matrix
+      end
+
+      # Compute greatest common divisor
+      #
+      # @param a [Integer]
+      # @param b [Integer]
+      # @return [Integer] GCD of a and b
+      def self.gcd(a, b)
+        return 0 if a.zero? && b.zero?
+        return a + b if a.zero? || b.zero?
+
+        while a.positive? && b.positive?
+          if a > b
+            a %= b
+          else
+            b %= a
+          end
+        end
+
+        a + b
+      end
+
+      # Verify algorithm by generating expected coefficients
+      #
+      # @param max_exponent [Integer] Maximum exponent to test
+      # @return [Hash<Integer, Integer>] Coefficient for each exponent
+      def self.generate_coefficient_table(max_exponent = 100)
+        bases = compute_bases(10) # Test with 10 data blocks
+        coefficients = {}
+
+        (0..max_exponent).each do |exponent|
+          # For par2cmdline, ALL data blocks use the same coefficient
+          # which is base[0] ^ exponent
+          coefficients[exponent] = Galois16.power(bases[0], exponent)
+        end
+
+        coefficients
+      end
+    end
+  end
+end

data/lib/omnizip/parity/par2cmdline_coefficients.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# Par2cmdline coefficient lookup table
+#
+# Discovered by reverse-engineering par2cmdline recovery blocks.
+# Par2cmdline uses the SAME coefficient for ALL data blocks for a given exponent.
+# This is different from standard Vandermonde matrix approach.
+#
+# Formula: coefficient[exponent, data_index] = PAR2CMDLINE_COEFFICIENTS[exponent]
+#          (same coefficient for all data_index values)
+
+module Omnizip
+  module Parity
+    # Par2cmdline-compatible coefficient table
+    PAR2CMDLINE_COEFFICIENTS = {
+      0 => 0x27c6,
+      1 => 0x8eb6,
+      2 => 0x1a9f,
+      3 => 0x2743,
+      4 => 0x24f6,
+      5 => 0x60d7,
+      6 => 0x2027,
+      7 => 0x1cf0,
+      8 => 0xd37a,
+      9 => 0xa961,
+      10 => 0xc6c7,
+      11 => 0x653e,
+      12 => 0x9c99,
+      13 => 0x2e1b,
+      14 => 0x8625,
+      15 => 0xd81e,
+      16 => 0x1fb5,
+      17 => 0x2cdd,
+      18 => 0x06ce,
+      19 => 0x41d5,
+      20 => 0xd297,
+      21 => 0xeae1,
+      22 => 0x9012,
+      23 => 0xdc31,
+      24 => 0xa33d,
+      25 => 0x2480,
+      26 => 0x3e2e,
+      27 => 0x5dee,
+      28 => 0x8f63,
+      29 => 0x90c5,
+      30 => 0xac21,
+      31 => 0x9bf4,
+      32 => 0x8b15,
+      33 => 0xc489,
+      34 => 0x004c,
+      35 => 0x6a45,
+      36 => 0x56f9,
+      37 => 0x6956,
+      38 => 0x2548,
+      39 => 0x0334,
+      40 => 0x3213,
+      41 => 0x7c7f,
+      42 => 0x1d3c,
+      43 => 0x9c1e,
+      44 => 0x835c,
+      45 => 0x7f30,
+      46 => 0x070e,
+      47 => 0x5f7d,
+      48 => 0x5f97,
+      49 => 0xfa32,
+      50 => 0x08fd,
+      51 => 0x9d43,
+      52 => 0x9ec1,
+      53 => 0x4643,
+      54 => 0x9222,
+      55 => 0x1f9c,
+      56 => 0xd271,
+      57 => 0xbd9f,
+      58 => 0xfef3,
+      59 => 0x9b83,
+    }.freeze
+  end
+end