omnizip 0.3.8 → 0.3.10

data/lib/omnizip/algorithms/lzma/match_finder.rb

@@ -26,15 +26,24 @@ module Omnizip
       # Match Finder using hash chain algorithm for LZ77 compression
       # Ported from XZ Utils lz_encoder.c
       class MatchFinder
-        HASH_SIZE = 4096
+        # Larger hash table for better distribution
+        # 16384 slots with 4-byte keys gives better collision resistance
+        HASH_SIZE = 16384
+        HASH_MASK = HASH_SIZE - 1
         MAX_MATCHES = 274
+        # Maximum hash chain depth to prevent O(n) lookups
+        # XZ Utils uses different values based on compression level (4-256)
+        # Ruby needs much lower values due to slower string operations
+        # Testing shows depth 16 gives same compression ratio as 32-64 but 2-3x faster
+        DEFAULT_CHAIN_DEPTH = 16
 
         attr_reader :dictionary, :buffer, :position
 
-        def initialize(dictionary)
+        def initialize(dictionary, chain_depth: DEFAULT_CHAIN_DEPTH)
           @dictionary = dictionary
           @buffer = String.new(encoding: Encoding::BINARY)
           @position = 0
+          @chain_depth = chain_depth
           # Use nil as empty marker (not 0) to distinguish from position 0
           @hash_table = Array.new(HASH_SIZE, nil)
           @hash_chain = Array.new(0)
@@ -84,38 +93,32 @@ module Omnizip
         # @return [Array<Match>] Array of matches sorted by length (descending)
         def find_matches(current_pos = @buffer.bytesize - 273)
           # Calculate hash for current position
+          # Note: Using 4-byte hash now (calc_hash checks pos + 4)
           hash = nil
-          if current_pos >= 0 && current_pos + 3 <= @buffer.bytesize
+          if current_pos >= 0 && current_pos + 4 <= @buffer.bytesize
             hash = calc_hash(@buffer, current_pos)
           end
 
-          # Update hash table for current position (even for early positions)
-          # This ensures positions 0-3 are available for later matches
-          # XZ Utils calls this "skip" - update hash without finding matches
-          # CRITICAL: Only update if this position hasn't been processed yet
-          # (i.e., @hash_table[hash] != current_pos)
-          # This prevents overwriting the hash chain when find_matches is called
-          # after skip() has already initialized the hash table
-          if hash && @hash_table[hash] != current_pos
-            @hash_chain[current_pos] = @hash_table[hash]
-            @hash_table[hash] = current_pos
-          end
+          # Skip hash table update here - skip() already built the table for all positions
+          # Adding current_pos to the chain is redundant since skip processed all positions
 
           # Can't find matches if no hash or insufficient data
           # Note: We CAN find matches at early positions (e.g., position 2 can match position 0)
-          # The only requirement is that there's enough data for hash calculation (current_pos + 3 <= buffer size)
+          # The only requirement is that there's enough data for hash calculation (current_pos + 4 <= buffer size)
           # and that there's at least 2 bytes of history (for MIN_MATCH_LENGTH=2)
           # CRITICAL: Don't produce matches until position >= 2 to ensure decoder has enough dict_full
           # The decoder validates: dict_full > distance, where dict_full starts at 0 after 1st byte
           # For distance=1 match to be valid, decoder needs dict_full >= 2 (at least 2 bytes decoded)
           # This happens after processing position 1 (first byte was literal at position 0)
           # So we can only produce matches starting at position 2
-          return [] if hash.nil? || @buffer.bytesize < 4 || current_pos + 3 > @buffer.bytesize || current_pos < 2
+          return [] if hash.nil? || @buffer.bytesize < 4 || current_pos + 4 > @buffer.bytesize || current_pos < 2
 
           @matches_count = 0
           chain_pos = @hash_chain[current_pos]
+          chain_depth = 0
 
-          while chain_pos && @matches_count < MAX_MATCHES
+          while chain_pos && @matches_count < MAX_MATCHES && chain_depth < @chain_depth
+            chain_depth += 1
             # CRITICAL: Skip invalid chain_pos values (beyond buffer or negative)
             next if chain_pos >= @buffer.bytesize || chain_pos.negative?
 
@@ -168,34 +171,61 @@ module Omnizip
 
         private
 
-        # Calculate hash for position (first 3 bytes)
+        # Calculate hash for position using 4-byte key with mixing
+        # Uses FNV-1a inspired mixing for better distribution
         #
         # @param data [String] Buffer data
         # @param pos [Integer] Position to hash
         # @return [Integer] Hash value
         def calc_hash(data, pos)
-          return 0 if pos + 3 > data.bytesize
-
-          (data.getbyte(pos) |
-           (data.getbyte(pos + 1) << 8) |
-           (data.getbyte(pos + 2) << 16)) % HASH_SIZE
+          return 0 if pos + 4 > data.bytesize
+
+          # Read 4 bytes as 32-bit integer (little-endian)
+          v = data.getbyte(pos) |
+            (data.getbyte(pos + 1) << 8) |
+            (data.getbyte(pos + 2) << 16) |
+            (data.getbyte(pos + 3) << 24)
+
+          # FNV-1a style mixing for better distribution
+          # Hash avalanche: each input bit affects all output bits
+          v = (v ^ (v >> 16)) * 0x45D9F3B
+          v = (v ^ (v >> 16)) * 0x45D9F3B
+          (v ^ (v >> 16)) & HASH_MASK
         end
 
         # Verify match length between two positions
+        # Uses 4-byte quick check then binary search with integer comparison
         #
         # @param pos1 [Integer] First position
         # @param pos2 [Integer] Second position
         # @return [Integer] Match length
         def verify_match(pos1, pos2)
-          max_len = [273, @buffer.bytesize - pos1, @buffer.bytesize - pos2].min
-          length = 0
+          buf = @buffer
+          max_len = [273, buf.bytesize - pos1, buf.bytesize - pos2].min
+          return 0 if max_len <= 0
+
+          # Quick rejection: check first 4 bytes as integer
+          if max_len >= 4
+            v1 = buf.byteslice(pos1, 4).unpack1("L<")
+            v2 = buf.byteslice(pos2, 4).unpack1("L<")
+            return 0 if v1 != v2
+          end
 
-          while length < max_len &&
-              @buffer.getbyte(pos1 + length) == @buffer.getbyte(pos2 + length)
-            length += 1
+          # Binary search for exact match length
+          # Creates O(log n) strings but does efficient integer comparison
+          low = 0
+          high = max_len
+
+          while low < high
+            mid = (low + high + 1) / 2
+            if buf.byteslice(pos1, mid) == buf.byteslice(pos2, mid)
+              low = mid
+            else
+              high = mid - 1
+            end
           end
 
-          length
+          low
         end
       end
     end

data/lib/omnizip/algorithms/lzma/optimal_encoder.rb

@@ -31,8 +31,6 @@ module Omnizip
           buf_pos = position
           buf_avail = [buf.bytesize - buf_pos, MATCH_LEN_MAX].min
 
-          # puts "[OPTIMUM] position=#{position} buf_avail=#{buf_avail} buf.bytesize=#{buf.bytesize}" if ENV['DEBUG']
-
           # Not enough input for a match
           return [UINT32_MAX, 1] if buf_avail < 2
 
@@ -51,26 +49,36 @@ module Omnizip
             # In Ruby: buf_back_index = buf_pos - rep_dist - 1
             buf_back_index = buf_pos - rep_dist - 1
 
-            # DEBUG: Trace repeated match check
-            # puts "[OPTIMAL] Checking rep#{i}: rep_dist=#{rep_dist} buf_pos=#{buf_pos} buf_back_index=#{buf_back_index}" if ENV['DEBUG']
-
             # Skip if out of bounds
             next if buf_back_index.negative? || buf_back_index >= buf.bytesize
 
-            # puts "[OPTIMAL] buf[#{buf_pos}]=#{'%02x' % buf.getbyte(buf_pos)} buf[#{buf_back_index}]=#{'%02x' % buf.getbyte(buf_back_index)}" if ENV['DEBUG']
-
             # If the first two bytes (2 == MATCH_LEN_MIN) do not match,
             # this rep is not useful.
             next if buf.getbyte(buf_pos) != buf.getbyte(buf_back_index) ||
               buf.getbyte(buf_pos + 1) != buf.getbyte(buf_back_index + 1)
 
             # The first two bytes matched. Calculate the length of the match.
+            # Use 64-bit integer comparison for speed (no string allocation)
+            max_match_len = [
+              buf.bytesize - buf_pos,
+              buf.bytesize - buf_back_index,
+              MATCH_LEN_MAX,
+              buf_avail,
+            ].min
+
             len = 2
-            while (buf_pos + len < buf.bytesize) &&
-                (buf_back_index + len < buf.bytesize) &&
-                (buf.getbyte(buf_pos + len) == buf.getbyte(buf_back_index + len)) &&
-                (len < MATCH_LEN_MAX) &&
-                (len < buf_avail)
+            # Compare 8 bytes at a time using 64-bit integers
+            while len + 8 <= max_match_len
+              # Read 8 bytes as little-endian 64-bit integers
+              v1 = buf.byteslice(buf_pos + len, 8).unpack1("Q<")
+              v2 = buf.byteslice(buf_back_index + len, 8).unpack1("Q<")
+              break if v1 != v2
+
+              len += 8
+            end
+            # Finalize with byte-by-byte for remaining bytes
+            while len < max_match_len &&
+                buf.getbyte(buf_pos + len) == buf.getbyte(buf_back_index + len)
               len += 1
             end