omnizip 0.4.7-x86_64-linux

data/lib/omnizip/algorithms/zstandard/huffman.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Huffman coding for Zstandard literals (RFC 8878 §4.2).
+      #
+      # Zstandard does NOT use symbol-ordered canonical Huffman. The
+      # decode table groups symbols by weight: all weight-1 symbols
+      # occupy the first DTable entries, then weight-2, and so on, with
+      # ascending symbol order inside each group. A symbol with weight
+      # w has code length (tableLog + 1 - w) and occupies (1 << w) >> 1
+      # consecutive DTable entries.
+      class Huffman
+        include Constants
+
+        DecodeEntry = Struct.new(:symbol, :nb_bits)
+
+        # @return [Array<Integer>] per-symbol weights (0 = absent)
+        attr_reader :weights
+
+        # @return [Integer] table log (max code length in this tree)
+        attr_reader :table_log
+
+        # Build a decode table from per-symbol weights.
+        #
+        # @param weights [Array<Integer>] one entry per symbol
+        # @return [Huffman]
+        def self.from_weights(weights)
+          table_log = compute_table_log(weights)
+          new(weights, table_log)
+        end
+
+        def initialize(weights, table_log)
+          @weights = weights
+          @table_log = table_log
+          @lookup = nil
+        end
+
+        # Decode one symbol from a reverse bitstream (peek max_bits,
+        # look up, consume the code's length).
+        #
+        # @param bitstream [FSE::BitStream]
+        # @return [Integer] decoded symbol byte
+        def decode(bitstream)
+          entry = lookup[bitstream.peek_bits(HUFFMAN_MAX_BITS)]
+          if entry.nil? || entry.nb_bits.zero?
+            raise Omnizip::DecompressionError,
+                  "Huffman lookup miss: no code for the next bits"
+          end
+
+          bitstream.read_bits(entry.nb_bits)
+          entry.symbol
+        end
+
+        # Build the flat 1 << HUFFMAN_MAX_BITS lookup table.
+        #
+        # @return [Array<DecodeEntry>]
+        def lookup
+          @lookup ||= begin
+            table_size = 1 << @table_log
+            dtable = Array.new(table_size) { DecodeEntry.new(0, 0) }
+            max_weight = @weights.max || 0
+
+            pos = 0
+            (1..max_weight).each do |w|
+              length = @table_log + 1 - w
+              entries_per_symbol = (1 << w) >> 1
+              @weights.each_with_index do |sw, sym|
+                next unless sw == w
+
+                entry = DecodeEntry.new(sym, length)
+                entries_per_symbol.times do
+                  if pos < table_size
+                    dtable[pos] = entry
+                    pos += 1
+                  end
+                end
+              end
+            end
+
+            expand = 1 << (HUFFMAN_MAX_BITS - @table_log)
+            lookup = []
+            table_size.times do |i|
+              expand.times { lookup << dtable[i] }
+            end
+            while lookup.length < (1 << HUFFMAN_MAX_BITS)
+              lookup << DecodeEntry.new(0, 0)
+            end
+            lookup
+          end
+        end
+
+        # Per-symbol (code, length) table for encoding, indexed by
+        # symbol value. Absent symbols map to (0, 0).
+        #
+        # @return [Array<Array(Integer, Integer)>]
+        def encode_table
+          @encode_table ||= begin
+            codes = Array.new(@weights.length) { [0, 0] }
+            max_weight = @weights.max || 0
+            dtable_pos = 0
+            (1..max_weight).each do |w|
+              length = @table_log + 1 - w
+              entries_per_symbol = (1 << w) >> 1
+              @weights.each_with_index do |sw, sym|
+                next unless sw == w
+
+                code = dtable_pos >> (@table_log - length)
+                codes[sym] = [code, length]
+                dtable_pos += entries_per_symbol
+              end
+            end
+            codes.fill([0, 0], codes.length..255)
+            codes
+          end
+        end
+
+        # Derive tableLog from the Kraft sum of the weights
+        # (C reference HUF_readStats).
+        #
+        # @return [Integer]
+        def self.compute_table_log(weights)
+          if weights.empty? || weights.all?(0)
+            raise Omnizip::DecompressionError,
+                  "Huffman weights: no present symbols"
+          end
+
+          weight_total = weights.sum { |w| w.positive? ? (1 << w) >> 1 : 0 }
+          if weight_total.zero?
+            raise Omnizip::DecompressionError, "Huffman weight total is 0"
+          end
+
+          table_log = if weight_total.nobits?(weight_total - 1)
+                        Constants.highbit32(weight_total)
+                      else
+                        Constants.highbit32(weight_total) + 1
+                      end
+          if table_log > HUFFMAN_MAX_BITS
+            raise Omnizip::DecompressionError,
+                  "Huffman tableLog #{table_log} exceeds max #{HUFFMAN_MAX_BITS}"
+          end
+
+          table_log
+        end
+      end
+
+      # Reads a Huffman tree description from the wire (RFC 8878
+      # §4.2.1) and builds a Huffman decode table.
+      class HuffmanTableReader
+        include Constants
+
+        # Read the table from the head of `src`.
+        #
+        # @param src [String]
+        # @return [Array(Huffman, Integer)] table and bytes consumed
+        def self.read(src)
+          raise Omnizip::DecompressionError, "empty Huffman header" if src.empty?
+
+          i_size = src.getbyte(0)
+          weights, consumed =
+            if i_size >= 128
+              read_direct_weights(src)
+            else
+              read_fse_compressed_weights(src)
+            end
+
+          [Huffman.from_weights(weights), consumed]
+        end
+
+        # Direct 4-bit weights: byte 0 is 127 + o_size, then
+        # (o_size + 1) / 2 bytes holding two nibbles each.
+        def self.read_direct_weights(src)
+          i_size = src.getbyte(0)
+          o_size = i_size - 127
+          packed_bytes = (o_size + 1) / 2
+          needed = 1 + packed_bytes
+          if src.bytesize < needed
+            raise Omnizip::DecompressionError,
+                  "truncated direct Huffman weights: need #{needed} bytes"
+          end
+
+          weights = []
+          (0...o_size).step(2) do |n|
+            byte = src.getbyte(1 + (n / 2))
+            weights << (byte >> 4)
+            weights << (byte & 0x0F) if n + 1 < o_size
+          end
+
+          weights << implied_last_weight(weights)
+          [weights, needed]
+        end
+
+        # FSE-compressed weights: byte 0 is the payload size, followed
+        # by an FSE table description and a 2-state interleaved
+        # bitstream (RFC 8878 §4.2.1.2).
+        def self.read_fse_compressed_weights(src)
+          i_size = src.getbyte(0)
+          if 1 + i_size > src.bytesize
+            raise Omnizip::DecompressionError,
+                  "truncated FSE Huffman weights: need #{1 + i_size} bytes"
+          end
+
+          payload = src.byteslice(1, i_size)
+          table, consumed = FSE.read_table(payload)
+          bitstream = payload.byteslice(consumed..)
+
+          weights = FSE.decode_stream(table, bitstream, HUF_SYMBOLVALUE_MAX)
+          weights << implied_last_weight(weights)
+          [weights, 1 + i_size]
+        end
+
+        # Compute the implied last weight via the Kraft inequality:
+        # sum(2^(w-1)) over all symbols equals 1 << tableLog.
+        def self.implied_last_weight(weights)
+          weight_total = 0
+          weights.each do |w|
+            if w > HUFFMAN_MAX_LOG
+              raise Omnizip::DecompressionError,
+                    "Huffman weight #{w} exceeds tableLog max"
+            end
+
+            weight_total += (1 << w) >> 1
+          end
+          if weight_total.zero?
+            raise Omnizip::DecompressionError, "Huffman weights sum to 0"
+          end
+
+          table_log = Constants.highbit32(weight_total) + 1
+          if table_log > HUFFMAN_MAX_LOG
+            raise Omnizip::DecompressionError,
+                  "Huffman tableLog #{table_log} exceeds max"
+          end
+
+          rest = (1 << table_log) - weight_total
+          if rest.zero?
+            raise Omnizip::DecompressionError,
+                  "Huffman weights already complete; no implied weight"
+          end
+          unless rest.nobits?(rest - 1)
+            raise Omnizip::DecompressionError,
+                  "Huffman implied weight remainder #{rest} is not a power of 2"
+          end
+
+          last_weight = Constants.highbit32(rest) + 1
+          if last_weight > HUFFMAN_MAX_LOG
+            raise Omnizip::DecompressionError,
+                  "Huffman implied weight #{last_weight} exceeds max"
+          end
+
+          last_weight
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/huffman_encoder.rb

@@ -0,0 +1,327 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Huffman encoder for Zstandard literals (RFC 8878 §4.2).
+      #
+      # Builds a length-limited Huffman code from per-byte frequencies,
+      # emits the weight table (direct or FSE-compressed), and codes
+      # the literals into 1 or 4 reverse bitstreams.
+      module HuffmanEncoder
+        include Constants
+
+        module_function
+
+        # Build 256 weights (0 for absent symbols) from the literal
+        # bytes, with code lengths capped at HUFFMAN_MAX_BITS.
+        #
+        # @param literals [String]
+        # @return [Array<Integer>]
+        def build_weights(literals)
+          counts = Array.new(256, 0)
+          literals.each_byte { |b| counts[b] += 1 }
+
+          present = (0..255).select { |b| counts[b].positive? }
+          if present.length < 2
+            # A Huffman table needs at least 2 symbols; pad with
+            # symbol 0 (or 1) so the real symbol keeps a real code.
+            weights = Array.new(256, 0)
+            sym = present.first || 0
+            weights[sym] = 1
+            other = sym.zero? ? 1 : 0
+            weights[other] = 1
+            return weights
+          end
+
+          freqs = present.map { |b| counts[b] }
+          lengths = huffman_lengths(freqs)
+          lengths = limit_lengths(lengths, HUFFMAN_MAX_BITS, freqs)
+
+          max_len = lengths.max
+          weights = Array.new(256, 0)
+          present.each_with_index do |byte, i|
+            weights[byte] = max_len - lengths[i] + 1
+          end
+          weights
+        end
+
+        # Standard Huffman code lengths via smallest-pair merging.
+        #
+        # @param freqs [Array<Integer>]
+        # @return [Array<Integer>] code length per symbol
+        def huffman_lengths(freqs)
+          nodes = freqs.map { |f| { freq: f, parent: -1 } }
+
+          while nodes.count { |n| n[:parent] == -1 } > 1
+            a = -1
+            b = -1
+            nodes.each_with_index do |n, i|
+              next unless n[:parent] == -1
+
+              if a == -1 || n[:freq] < nodes[a][:freq]
+                b = a
+                a = i
+              elsif b == -1 || n[:freq] < nodes[b][:freq]
+                b = i
+              end
+            end
+
+            nodes[a][:parent] = nodes.length
+            nodes[b][:parent] = nodes.length
+            nodes << { freq: nodes[a][:freq] + nodes[b][:freq], parent: -1 }
+          end
+
+          lengths = Array.new(freqs.length, 0)
+          freqs.each_index do |i|
+            len = 0
+            cur = i
+            while nodes[cur][:parent] != -1
+              cur = nodes[cur][:parent]
+              len += 1
+            end
+            lengths[i] = [len, 255].min
+          end
+          lengths
+        end
+
+        # Cap code lengths at max_len and repair the Kraft sum to be
+        # EXACTLY 2^max_len, in integer arithmetic. An exact sum is
+        # what lets the decoder re-derive the dropped last weight as a
+        # clean power of two — a float-tolerance repair can leave the
+        # remainder non-power-of-two and the frame undecodable.
+        #
+        # @param lengths [Array<Integer>] mutated in place
+        # @param max_len [Integer]
+        # @param freqs [Array<Integer>]
+        # @return [Array<Integer>]
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable-next Metrics/AbcSize
+        def limit_lengths(lengths, max_len, freqs)
+          lengths.map! { |l| [l, max_len].min }
+
+          target = 1 << max_len
+          kraft = lengths.sum { |l| l.positive? ? 1 << (max_len - l) : 0 }
+
+          # Over-subscribed: lengthen the least-frequent shortest
+          # codes until the sum fits.
+          while kraft > target
+            idx = kraft_adjust_index(lengths, freqs) do |l|
+              l.positive? && l < max_len
+            end
+            lengths[idx] += 1
+            kraft -= 1 << (max_len - lengths[idx])
+          end
+
+          # Under-subscribed: shorten the most-frequent code that fits
+          # without overshooting; if none fits, lengthen one and let
+          # the first loop re-fit.
+          while kraft < target
+            best = nil
+            lengths.each_with_index do |l, i|
+              next unless l > 1
+
+              contribution = 1 << (max_len - l)
+              next if kraft + contribution > target
+              next if best && freqs[i] <= freqs[best]
+
+              best = i
+            end
+            if best.nil?
+              idx = kraft_adjust_index(lengths, freqs) do |l|
+                l.positive? && l < max_len
+              end
+              lengths[idx] += 1
+              kraft -= 1 << (max_len - lengths[idx])
+            else
+              delta = 1 << (max_len - lengths[best])
+              lengths[best] -= 1
+              kraft += delta
+            end
+          end
+
+          lengths
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        # Index of the least-frequent symbol matching the block.
+        def kraft_adjust_index(lengths, freqs)
+          best = nil
+          lengths.each_with_index do |l, i|
+            next unless yield(l)
+            next if best && freqs[i] >= freqs[best]
+
+            best = i
+          end
+          best
+        end
+
+        # Serialize the weight table. Direct 4-bit weights when the
+        # alphabet fits, FSE-compressed otherwise.
+        #
+        # @param weights [Array<Integer>]
+        # @return [String]
+        def encode_weights(weights)
+          max_symbol = weights.rindex(&:positive?)
+          raise Omnizip::CompressionError, "no present Huffman weights" if max_symbol.nil?
+
+          if max_symbol <= 128
+            encode_weights_direct(weights, max_symbol)
+          else
+            encode_weights_fse(weights, max_symbol)
+          end
+        end
+
+        # Direct encoding: header byte 127 + o_size, then two 4-bit
+        # weights per byte. The last present weight is implied on
+        # decode and is dropped here.
+        def encode_weights_direct(weights, max_symbol)
+          o_size = max_symbol
+          i_size = 127 + o_size
+          out = [i_size].pack("C")
+
+          (0...o_size).step(2) do |n|
+            high = weights[n] & 0x0F
+            low = n + 1 < o_size ? weights[n + 1] & 0x0F : 0
+            out << ((high << 4) | low).chr
+          end
+
+          out
+        end
+
+        # FSE-compressed weights (RFC 8878 §4.2.1.2): payload-size
+        # header byte, NCount, then the 2-state bitstream.
+        def encode_weights_fse(weights, max_symbol)
+          o_size = max_symbol
+          symbols = weights.first(o_size)
+
+          distinct = symbols.uniq.length
+          if distinct <= 1
+            raise Omnizip::CompressionError,
+                  "uniform Huffman weights give no compression"
+          end
+
+          encoder = FSE::Encoder.build_from_symbols(symbols, 11, 6)
+          if encoder.nil?
+            raise Omnizip::CompressionError,
+                  "uniform Huffman weights give no compression"
+          end
+
+          payload = encoder.compress(symbols)
+          if payload.bytesize >= 128
+            raise Omnizip::CompressionError,
+                  "FSE weight payload exceeds the 127-byte header limit"
+          end
+
+          [payload.bytesize].pack("C") + payload
+        end
+
+        # Encode literals as a Compressed_Literals_Block section:
+        # header + weights + coded stream(s).
+        #
+        # @param literals [String]
+        # @return [String]
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable-next Metrics/AbcSize
+        def encode_literals(literals)
+          weights = build_weights(literals)
+          # The wire format drops the last present weight (it is implied
+          # by the Kraft inequality on decode), so the coding table must
+          # be rebuilt exactly the way the decoder rebuilds it.
+          max_symbol = weights.rindex(&:positive?)
+          wire_weights = weights.first(max_symbol)
+          full_weights = wire_weights +
+            [HuffmanTableReader.implied_last_weight(wire_weights)]
+          table = Huffman.from_weights(full_weights)
+          encode_table = table.encode_table
+
+          weights_wire = encode_weights(weights)
+          lit_size = literals.bytesize
+
+          # Single stream (3-byte header) when both sizes fit 10 bits.
+          if lit_size < 1024
+            coded = encode_huffman_stream(encode_table, literals)
+            lit_c_size = weights_wire.bytesize + coded.bytesize
+            if lit_c_size < 1024
+              header = LITERALS_BLOCK_COMPRESSED |
+                (lit_size << 4) | (lit_c_size << 14)
+              return [header].pack("V")[0, 3] + weights_wire + coded
+            end
+          end
+
+          # 4 streams with a jump table.
+          segment_size = (lit_size + 3) / 4
+          segments = Array.new(4) do |i|
+            encode_huffman_stream(
+              encode_table,
+              literals.byteslice(segment_size * i, segment_size),
+            )
+          end
+
+          lit_c_size = weights_wire.bytesize + 6 + segments.sum(&:bytesize)
+
+          out = if lit_size < 1024 && lit_c_size < 1024
+                  header = LITERALS_BLOCK_COMPRESSED | (0b01 << 2) |
+                    (lit_size << 4) | (lit_c_size << 14)
+                  [header].pack("V")[0, 3]
+                elsif lit_size < 16_384 && lit_c_size < 16_384
+                  header = LITERALS_BLOCK_COMPRESSED | (0b10 << 2) |
+                    (lit_size << 4) | (lit_c_size << 18)
+                  [header].pack("V")[0, 4]
+                elsif lit_size < 262_144 && lit_c_size < 262_144
+                  low = LITERALS_BLOCK_COMPRESSED | (0b11 << 2) |
+                    (lit_size << 4) | ((lit_c_size & 0x3FF) << 22)
+                  [low].pack("V") + [(lit_c_size >> 10) & 0xFF].pack("C")
+                else
+                  raise Omnizip::CompressionError,
+                        "literals section exceeds the 18-bit header limits"
+                end
+
+          out + weights_wire +
+            segments.first(3).map { |s| [s.bytesize].pack("v") }.join +
+            segments.join
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        # Code literals into one reverse bitstream (C BIT_CStream
+        # direction): the encoder writes the last symbol first so the
+        # reverse reader recovers symbols in order.
+        #
+        # @param encode_table [Array<Array(Integer, Integer)>]
+        # @param literals [String]
+        # @return [String]
+        def encode_huffman_stream(encode_table, literals)
+          bitc = FSE::Encoder::BitCStream.new
+          (literals.bytesize - 1).downto(0) do |i|
+            code, len = encode_table[literals.getbyte(i)]
+            next if len.zero?
+
+            bitc.add_bits(code, len)
+            bitc.flush
+          end
+          bitc.close
+        end
+      end
+    end
+  end
+end