omnizip 0.3.13 → 0.3.15

data/lib/omnizip/algorithms/zstandard/literals.rb

@@ -23,150 +23,217 @@
 module Omnizip
   module Algorithms
     class Zstandard
-      # Literals section decoder (RFC 8878 Section 3.1.1.3.1)
+      # Literals section decoder (RFC 8878 §3.1.1.3.1).
       #
-      # Decodes the literals section of a compressed block.
-      # Can be raw, RLE, Huffman compressed, or treeless.
+      # Header byte 0 layout (bits numbered from the LSB, matching the C
+      # reference `litEncType = istart[0] & 3`):
+      #
+      #   bits 0-1  Literals_Block_Type (0=Raw, 1=RLE, 2=Compressed,
+      #             3=Treeless)
+      #   bits 2-3  Size_Format selector
+      #
+      # Raw / RLE size formats:
+      #   00, 10 -> 1-byte header, regen = byte0 >> 3
+      #   01     -> 2-byte header, regen = LE16 >> 4
+      #   11     -> 3-byte header, regen = LE24 >> 4
+      #
+      # Compressed / Treeless size formats:
+      #   00 -> 3-byte header, single stream, 10-bit sizes
+      #   01 -> 3-byte header, 4 streams, 10-bit sizes
+      #   10 -> 4-byte header, 4 streams, 14-bit sizes
+      #   11 -> 5-byte header, 4 streams, 18-bit sizes
       class LiteralsDecoder
         include Constants
 
-        # @return [String] Decoded literals
+        # @return [String] decoded literal bytes
         attr_reader :literals
 
-        # @return [Huffman, nil] Huffman table for future treeless blocks
+        # @return [Huffman, nil] table from a Compressed block, for the
+        #   next Treeless block in the same frame
         attr_reader :huffman_table
 
-        # Parse and decode literals section
-        #
-        # @param input [IO] Input stream positioned at literals section
-        # @param previous_table [Huffman, nil] Previous Huffman table (for treeless)
-        # @return [LiteralsDecoder] Decoder with decoded literals
-        def self.decode(input, previous_table = nil)
-          decoder = new(input, previous_table)
-          decoder.decode_section
-          decoder
-        end
+        # @return [Integer] bytes consumed from the input
+        attr_reader :consumed
 
-        # Initialize decoder
+        # Decode the literals section at the head of `input`.
         #
-        # @param input [IO] Input stream
-        # @param previous_table [Huffman, nil] Previous Huffman table
-        def initialize(input, previous_table = nil)
-          @input = input
-          @previous_table = previous_table
-          @literals = String.new(encoding: Encoding::BINARY)
-          @huffman_table = previous_table
-        end
+        # @param input [String]
+        # @param previous_table [Huffman, nil] table from the previous
+        #   compressed literals block (required for Treeless)
+        # @return [LiteralsDecoder]
+        def self.decode(input, previous_table = nil)
+          raise Omnizip::DecompressionError, "empty literals section" if input.empty?
 
-        # Decode the literals section
-        #
-        # @return [void]
-        def decode_section
-          # Read literals header (1-3 bytes)
-          header1 = @input.read(1).ord
-          block_type = (header1 >> 6) & 0x03
+          block_type = input.getbyte(0) & 0x03
 
           case block_type
-          when LITERALS_BLOCK_RAW
-            decode_raw(header1)
-          when LITERALS_BLOCK_RLE
-            decode_rle(header1)
+          when LITERALS_BLOCK_RAW then decode_raw(input)
+          when LITERALS_BLOCK_RLE then decode_rle(input)
           when LITERALS_BLOCK_COMPRESSED
-            decode_compressed(header1)
+            decode_compressed(input, previous_table, false)
           when LITERALS_BLOCK_TREELESS
-            decode_treeless(header1)
+            decode_compressed(input, previous_table, true)
           end
         end
 
-        private
+        def self.decode_raw(input)
+          regen_size, header_size = size_format_raw_rle(input)
+          end_pos = header_size + regen_size
+          if input.bytesize < end_pos
+            raise Omnizip::DecompressionError,
+                  "truncated raw literals: need #{end_pos} bytes"
+          end
 
-        # Decode raw (uncompressed) literals
-        def decode_raw(header1)
-          # Size format: 5-bit or 12-bit or 20-bit
-          size = header1 & 0x1F
+          new(input.byteslice(header_size, regen_size), nil, end_pos)
+        end
 
-          if size == 31
-            # Read 2 more bytes for 12-bit size
-            header2 = @input.read(2).unpack1("v")
-            size = header2 + 31
+        def self.decode_rle(input)
+          regen_size, header_size = size_format_raw_rle(input)
+          if input.bytesize < header_size + 1
+            raise Omnizip::DecompressionError,
+                  "truncated RLE literals: missing repeated byte"
           end
 
-          @literals = @input.read(size)
+          byte = input.getbyte(header_size)
+          new(byte.chr * regen_size, nil, header_size + 1)
         end
 
-        # Decode RLE (run-length encoded) literals
-        def decode_rle(header1)
-          # Size format: 5-bit or 12-bit
-          size = header1 & 0x1F
-
-          if size == 31
-            # Read 2 more bytes for 12-bit size
-            header2 = @input.read(2).unpack1("v")
-            size = header2 + 31
+        # rubocop:disable Metrics/AbcSize
+        # rubocop:disable-next Metrics/MethodLength
+        def self.decode_compressed(input, previous_table, is_repeat)
+          if is_repeat && previous_table.nil?
+            raise Omnizip::DecompressionError,
+                  "treeless literals block requires a previous compressed " \
+                  "block in the same frame"
           end
 
-          # Read single byte and repeat
-          byte = @input.read(1)
-          @literals = byte * size
-        end
+          lhl_code = (input.getbyte(0) >> 2) & 0x03
 
-        # Decode Huffman-compressed literals
-        def decode_compressed(header1)
-          # Read regenerated size (5-bit or 12-bit or 20-bit)
-          size = header1 & 0x1F
-          1
-
-          if size == 31
-            # Check next byte
-            header2 = @input.read(1).ord
-            if header2 < 128
-              # 12-bit size
-              header3 = @input.read(1).ord
-              size = (header2 | (header3 << 7)) + 31
-              3
+          min_header = [3, 3, 4, 5][lhl_code]
+          if input.bytesize < min_header
+            raise Omnizip::DecompressionError,
+                  "truncated compressed literals header: need #{min_header} bytes"
+          end
+
+          lit_size, lit_c_size, lh_size, single_stream =
+            case lhl_code
+            when 0, 1
+              lhc = input.getbyte(0) |
+                (input.getbyte(1) << 8) | (input.getbyte(2) << 16)
+              [(lhc >> 4) & 0x3FF, (lhc >> 14) & 0x3FF, 3, lhl_code.zero?]
+            when 2
+              lhc = input.unpack1("V") & 0xFFFFFFFF
+              [(lhc >> 4) & 0x3FFF, lhc >> 18, 4, false]
             else
-              # 20-bit size
-              header3 = @input.read(3)
-              size = ((header2 & 0x7F) | (header3.unpack1("V") << 7)) + 31
-              4
+              low = input.byteslice(0, 4).unpack1("V")
+              high = input.getbyte(4)
+              lhc = low | (high << 32)
+              [(lhc >> 4) & 0x3FFFF, lhc >> 22, 5, false]
             end
-          end
 
-          regenerated_size = size
+          needed = lh_size + lit_c_size
+          if input.bytesize < needed
+            raise Omnizip::DecompressionError,
+                  "truncated compressed literals: need #{needed} bytes"
+          end
+          compressed = input.byteslice(lh_size, lit_c_size)
 
-          # Read compressed size (if block type is compressed)
-          # Actually, for LITERALS_BLOCK_COMPRESSED, we need to read compressed size
-          # The format is more complex - let's simplify
+          table_bytes = 0
+          table = previous_table
+          unless is_repeat
+            table, table_bytes = HuffmanTableReader.read(compressed)
+          end
 
-          # Read Huffman table
-          @huffman_table = HuffmanTableReader.read(@input)
+          literal_data = compressed.byteslice(table_bytes..)
+          literals = if single_stream
+                       decode_single_stream(table, literal_data, lit_size)
+                     else
+                       decode_four_stream(table, literal_data, lit_size)
+                     end
 
-          # For simplicity, just read raw bytes (full Huffman decoding is complex)
-          # This is a simplified implementation
-          @literals = @input.read(regenerated_size)
+          new(literals, is_repeat ? nil : table, needed)
         end
+        # rubocop:enable Metrics/AbcSize
+
+        # rubocop:disable-next Metrics/AbcSize
+        def self.size_format_raw_rle(input)
+          lhl_code = (input.getbyte(0) >> 2) & 0x03
+          header_size = [1, 2, 1, 3][lhl_code]
+          if input.bytesize < header_size
+            raise Omnizip::DecompressionError,
+                  "truncated Raw/RLE literals header: need #{header_size} bytes"
+          end
 
-        # Decode treeless literals (reuse previous Huffman table)
-        def decode_treeless(header1)
-          # Similar to compressed but without Huffman table
-          size = header1 & 0x1F
+          case lhl_code
+          when 0, 2
+            [input.getbyte(0) >> 3, 1]
+          when 1
+            [input.unpack1("v") >> 4, 2]
+          else
+            lhc = input.getbyte(0) | (input.getbyte(1) << 8) |
+              (input.getbyte(2) << 16)
+            [lhc >> 4, 3]
+          end
+        end
 
-          if size == 31
-            header2 = @input.read(2).unpack1("v")
-            size = header2 + 31
+        # One reverse bitstream decoded into lit_size symbols.
+        def self.decode_single_stream(table, src, lit_size)
+          bs = FSE::BitStream.new(src)
+          out = String.new(encoding: Encoding::BINARY)
+          lit_size.times do
+            out << table.decode(bs).chr
+            bs.reload
           end
+          out
+        end
 
-          regenerated_size = size
+        # Jump table + four independent reverse bitstreams, each decoding
+        # ~1/4 of the literals (RFC 8878 §3.1.1.3.1.4).
+        # rubocop:disable Metrics/AbcSize
+        def self.decode_four_stream(table, src, lit_size)
+          if src.bytesize < 10
+            raise Omnizip::DecompressionError,
+                  "4-stream literals too short (need at least 10 bytes)"
+          end
+          # rubocop:enable Metrics/AbcSize
+
+          length1 = src.unpack1("v")
+          length2 = src.byteslice(2, 2).unpack1("v")
+          length3 = src.byteslice(4, 2).unpack1("v")
+          total = src.bytesize - 6
+          if length1 + length2 + length3 > total
+            raise Omnizip::DecompressionError,
+                  "4-stream jump table sizes exceed total stream size"
+          end
 
-          # Use previous Huffman table
-          if @previous_table.nil?
-            # No previous table - this is an error
-            @literals = @input.read(regenerated_size)
-            return
+          streams = src.byteslice(6..)
+          seg1 = streams.byteslice(0, length1)
+          seg2 = streams.byteslice(length1, length2)
+          seg3 = streams.byteslice(length1 + length2, length3)
+          seg4 = streams.byteslice((length1 + length2 + length3)..)
+
+          segment_size = (lit_size + 3) / 4
+          bounds = [0,
+                    segment_size,
+                    segment_size * 2,
+                    segment_size * 3,
+                    lit_size]
+
+          out = String.new(encoding: Encoding::BINARY)
+          [seg1, seg2, seg3, seg4].each_with_index do |seg, i|
+            bs = FSE::BitStream.new(seg)
+            (bounds[i]...bounds[i + 1]).each do
+              out << table.decode(bs).chr
+              bs.reload
+            end
           end
+          out
+        end
 
-          # For simplicity, just read raw bytes
-          @literals = @input.read(regenerated_size)
+        def initialize(literals, huffman_table, consumed)
+          @literals = literals
+          @huffman_table = huffman_table
+          @consumed = consumed
         end
       end
     end

data/lib/omnizip/algorithms/zstandard/literals_encoder.rb

@@ -23,225 +23,73 @@
 module Omnizip
   module Algorithms
     class Zstandard
-      # Literals Section Encoder (RFC 8878 Section 3.1.1.3.1)
-      #
-      # Encodes literals sections for Zstandard compressed blocks.
-      # Supports raw, RLE, and Huffman-compressed literals.
-      class LiteralsEncoder
+      # Literals Section encoder (RFC 8878 §3.1.1.3.1): picks Raw, RLE
+      # or Huffman-compressed, whichever is smallest.
+      module LiteralsEncoder
         include Constants
 
-        # @return [HuffmanEncoder, nil] Huffman encoder for this block
-        attr_reader :huffman_encoder
+        module_function
 
-        # Encode literals section
+        # Encode a literals section for `literals`.
         #
-        # @param literals [String] Literal bytes to encode
-        # @param previous_huffman [HuffmanEncoder, nil] Previous Huffman encoder (for treeless)
-        # @param use_compression [Boolean] Whether to use Huffman compression
-        # @return [String] Encoded literals section
-        def self.encode(literals, previous_huffman: nil, use_compression: true)
-          encoder = new(literals, previous_huffman, use_compression)
-          encoder.encode_section
-        end
-
-        # Initialize literals encoder
-        #
-        # @param literals [String] Literal bytes
-        # @param previous_huffman [HuffmanEncoder, nil] Previous Huffman encoder
-        # @param use_compression [Boolean] Whether to use compression
-        def initialize(literals, previous_huffman = nil, use_compression = true)
-          @literals = literals.to_s.dup.force_encoding(Encoding::BINARY)
-          @previous_huffman = previous_huffman
-          @use_compression = use_compression
-          @huffman_encoder = nil
-        end
-
-        # Encode the literals section
-        #
-        # @return [String] Encoded section
-        def encode_section
-          return encode_empty if @literals.empty?
-
-          # Choose encoding method based on data characteristics
-          if rle_efficient?
-            encode_rle
-          elsif @use_compression && huffman_efficient?
-            encode_huffman
-          else
-            encode_raw
+        # @param literals [String]
+        # @return [String] the full section (header + content)
+        def encode(literals)
+          return encode_raw(literals) if literals.bytesize < 16
+
+          return encode_rle(literals) if rle?(literals)
+
+          compressed = nil
+          begin
+            compressed = HuffmanEncoder.encode_literals(literals)
+          rescue Omnizip::CompressionError
+            compressed = nil
           end
-        end
-
-        private
-
-        # Check if RLE encoding would be efficient
-        def rle_efficient?
-          return false if @literals.length < 3
-
-          # Check if all bytes are the same
-          first_byte = @literals.getbyte(0)
-          @literals.bytes.all?(first_byte)
-        end
-
-        # Check if Huffman encoding would be efficient
-        def huffman_efficient?
-          return false if @literals.length < 16
-
-          # Check if data has enough redundancy
-          entropy = calculate_entropy(@literals)
-          entropy < 7.5 # Less than 7.5 bits per byte suggests compressibility
-        end
-
-        # Calculate Shannon entropy of data
-        def calculate_entropy(data)
-          return 0 if data.empty?
-
-          # Count byte frequencies
-          freq = Array.new(256, 0)
-          data.each_byte { |b| freq[b] += 1 }
 
-          # Calculate entropy
-          total = data.length.to_f
-          entropy = 0.0
-
-          freq.each do |count|
-            next if count.zero?
-
-            prob = count / total
-            entropy -= prob * Math.log2(prob)
+          if compressed && compressed.bytesize < literals.bytesize
+            compressed
+          else
+            encode_raw(literals)
           end
-
-          entropy
-        end
-
-        # Encode empty literals
-        def encode_empty
-          # Type 0 (raw), size 0
-          "\x00"
         end
 
-        # Encode raw (uncompressed) literals
-        def encode_raw
-          size = @literals.bytesize
-          header = encode_literals_header(LITERALS_BLOCK_RAW, size)
-          header + @literals
+        # Raw literals block with the minimal size-format header.
+        def encode_raw(literals)
+          encode_raw_rle_header(LITERALS_BLOCK_RAW, literals.bytesize) +
+            literals
         end
 
-        # Encode RLE (run-length encoded) literals
-        def encode_rle
-          size = @literals.bytesize
-          byte = @literals.getbyte(0)
-
-          header = encode_literals_header(LITERALS_BLOCK_RLE, size)
-          header + [byte].pack("C")
-        end
-
-        # Encode Huffman-compressed literals
-        def encode_huffman
-          size = @literals.bytesize
-
-          # Build Huffman tree from literals
-          @huffman_encoder = build_huffman_encoder(@literals)
-
-          if @huffman_encoder.nil?
-            # Fallback to raw if Huffman fails
-            return encode_raw
-          end
-
-          # Encode literals with Huffman
-          compressed = @huffman_encoder.encode(@literals)
-
-          # Check if compression is beneficial
-          # Need to account for header + table description overhead
-          table_desc = @huffman_encoder.encode_table_description
-          total_compressed_size = compressed.bytesize + table_desc.bytesize
-
-          if total_compressed_size >= size
-            # Not beneficial, use raw
-            @huffman_encoder = nil
-            return encode_raw
-          end
-
-          # Build header for LITERALS_BLOCK_COMPRESSED
-          # Type (2 bits) = 10, followed by regenerated size
-          header = encode_literals_header(LITERALS_BLOCK_COMPRESSED, size,
-                                          total_compressed_size)
-
-          # Build complete section: header + table_desc + compressed
-          header + table_desc + compressed
+        # RLE literals block: one byte, repeated.
+        def encode_rle(literals)
+          encode_raw_rle_header(LITERALS_BLOCK_RLE, literals.bytesize) +
+            literals.getbyte(0).chr
         end
 
-        # Encode literals header according to RFC 8878 Section 3.1.1.3.1
-        #
-        # For compressed blocks:
-        # - Type (2 bits) in first byte
-        # - Regenerated size (variable length)
-        # - Compressed size (variable length, only for compressed type)
-        def encode_literals_header(type, regenerated_size,
-compressed_size = nil)
-          # Encode regenerated size
-          if regenerated_size < 32
-            # 5-bit size: type(2) + size(5) + padding(1) = 8 bits
-            header_byte = (type << 6) | regenerated_size
-            header = [header_byte].pack("C")
-          elsif regenerated_size < 4096
-            # 12-bit size
-            header_byte = (type << 6) | 31
-            size_field = regenerated_size - 31
-            header = [header_byte, size_field & 0xFF,
-                      (size_field >> 8) & 0xFF].pack("Cv")
-          else
-            # 20-bit size
-            header_byte = (type << 6) | 31
-            # Extended size format
-            header = [header_byte].pack("C")
-            header += encode_extended_size(regenerated_size - 31)
-          end
-
-          # Add compressed size for LITERALS_BLOCK_COMPRESSED
-          if type == LITERALS_BLOCK_COMPRESSED && compressed_size
-            header + encode_compressed_size(compressed_size)
-          else
-            header
-          end
-        end
+        def rle?(literals)
+          first = literals.getbyte(0)
+          idx = 1
+          while idx < literals.bytesize
+            return false if literals.getbyte(idx) != first
 
-        # Encode extended size (20-bit or more)
-        def encode_extended_size(size)
-          if size < 128
-            # Single byte
-            [size].pack("C")
-          elsif size < 16384
-            # Two bytes
-            [size | 0x80, (size >> 7) & 0x7F].pack("CC")
-          else
-            # Three bytes
-            [size | 0x80, (size >> 7) | 0x80, (size >> 14) & 0x7F].pack("CCC")
+            idx += 1
           end
+          true
         end
 
-        # Encode compressed size
-        def encode_compressed_size(size)
-          if size < 128
-            [size].pack("C")
-          elsif size < 16384
-            [size | 0x80, (size >> 7) & 0x7F].pack("CC")
+        # 1/2/3-byte Raw/RLE header per the size-format rules. For the
+        # 2- and 3-byte forms the Size_Format bits (01 / 11) live in
+        # bits 2-3 and the size starts at bit 4.
+        def encode_raw_rle_header(type, size)
+          if size < 32
+            [type | (size << 3)].pack("C")
+          elsif size < 4096
+            lhc = type | 0x04 | (size << 4)
+            [lhc & 0xFF, (lhc >> 8) & 0xFF].pack("CC")
           else
-            [size | 0x80, (size >> 7) | 0x80, (size >> 14) & 0x7F].pack("CCC")
+            lhc = type | 0x0C | (size << 4)
+            [lhc & 0xFF, (lhc >> 8) & 0xFF, (lhc >> 16) & 0xFF].pack("CCC")
           end
         end
-
-        # Build Huffman encoder from data
-        def build_huffman_encoder(data)
-          return nil if data.nil? || data.empty?
-
-          # Count byte frequencies
-          freq = Array.new(256, 0)
-          data.each_byte { |b| freq[b] += 1 }
-
-          # Build Huffman encoder
-          HuffmanEncoder.build_from_frequencies(freq, HUFFMAN_MAX_BITS)
-        end
       end
     end
   end