jxl 1.0.0

data/lib/jxl/entropy/ans_distribution.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module JXL
+  module Entropy
+    module Histogram
+      LOG_RANGE = 12
+      RANGE = 1 << LOG_RANGE
+      HUFFMAN_BLOCK = [[3, 10], [7, 12], [3, 7], [4, 3], [3, 6], [3, 8], [3, 9], [4, 5],
+                       [3, 10], [4, 4], [3, 7], [4, 1], [3, 6], [3, 8], [3, 9], [4, 2]].freeze
+      LOG_COUNT_HUFFMAN = [HUFFMAN_BLOCK,
+                           HUFFMAN_BLOCK.dup.tap { |b| b[1] = [5, 0] },
+                           HUFFMAN_BLOCK.dup.tap { |b| b[1] = [6, 11] },
+                           HUFFMAN_BLOCK.dup.tap { |b| b[1] = [5, 0] },
+                           HUFFMAN_BLOCK.dup.tap { |b| b[1] = [7, 13] },
+                           HUFFMAN_BLOCK.dup.tap { |b| b[1] = [5, 0] },
+                           HUFFMAN_BLOCK.dup.tap { |b| b[1] = [6, 11] },
+                           HUFFMAN_BLOCK.dup.tap { |b| b[1] = [5, 0] }].flatten(1).freeze
+
+      module_function
+
+      def read(reader)
+        return read_simple(reader) unless reader.read(1).zero?
+        return flat(varlen8(reader) + 1) unless reader.read(1).zero?
+
+        read_general(reader)
+      end
+
+      def read_simple(reader)
+        symbols = Array.new(reader.read(1) + 1) { varlen8(reader) }
+        raise CorruptError, "duplicate histogram symbol" unless symbols.uniq.length == symbols.length
+        return Array.new(symbols.first + 1, 0).tap { |counts| counts[symbols.first] = RANGE } if symbols.one?
+
+        counts = Array.new(symbols.max + 1, 0)
+        counts[symbols[0]] = reader.read(LOG_RANGE)
+        counts[symbols[1]] = RANGE - counts[symbols[0]]
+        raise CorruptError, "empty histogram symbol" if counts.any?(&:negative?)
+
+        counts
+      end
+      private_class_method :read_simple
+
+      def flat(length)
+        raise CorruptError, "histogram alphabet is too large" unless length.between?(1, RANGE)
+
+        quotient, remainder = RANGE.divmod(length)
+        Array.new(length) { |index| quotient + (index < remainder ? 1 : 0) }
+      end
+
+      def read_general(reader) # rubocop:disable Metrics/AbcSize
+        shift = read_shift(reader)
+        length = varlen8(reader) + 3
+        counts = Array.new(length, 0)
+        log_counts = Array.new(length, 0)
+        same = Array.new(length, 0)
+        omit_log = -1
+        omit_position = -1
+        index = 0
+        while index < length
+          bits, symbol = LOG_COUNT_HUFFMAN.fetch(reader.peek(7))
+          reader.skip(bits)
+          log_counts[index] = symbol - 1
+          if log_counts[index] == LOG_RANGE
+            run = varlen8(reader)
+            same[index] = run + 5
+            index += run + 4
+            next
+          end
+          if log_counts[index] > omit_log
+            omit_log = log_counts[index]
+            omit_position = index
+          end
+          index += 1
+        end
+        if omit_position.negative? || (omit_position + 1 < length && log_counts[omit_position + 1] == LOG_RANGE)
+          raise CorruptError, "invalid histogram"
+        end
+
+        fill_counts(reader, counts, log_counts, same, omit_position, shift)
+      end
+      private_class_method :read_general
+
+      # rubocop:disable-next Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/ParameterLists,Metrics/PerceivedComplexity
+      def fill_counts(reader, counts, log_counts, same, omit_position, shift)
+        total = 0
+        previous = 0
+        remaining = 0
+        counts.each_index do |index|
+          if same[index].positive?
+            remaining = same[index] - 1
+            previous = index.positive? ? counts[index - 1] : 0
+          end
+          if remaining.positive?
+            counts[index] = previous
+            remaining -= 1
+          else
+            code = log_counts[index]
+            next if index == omit_position || code.negative?
+
+            precision = population_precision(code, shift)
+            counts[index] = (1 << code) + (reader.read(precision) << (code - precision))
+          end
+          total += counts[index]
+        end
+        counts[omit_position] = RANGE - total
+        raise CorruptError, "histogram counts exceed range" unless counts[omit_position].positive?
+
+        counts
+      end
+      private_class_method :fill_counts
+
+      def read_shift(reader)
+        log = 0
+        log += 1 while log < 12 && !reader.read(1).zero?
+        shift = (reader.read(log) | (1 << log)) - 1
+        raise CorruptError, "invalid histogram shift" if shift > LOG_RANGE + 1
+
+        shift
+      end
+      private_class_method :read_shift
+
+      def population_precision(log_count, shift)
+        [log_count, shift - ((LOG_RANGE - log_count) >> 1)].min.clamp(0, log_count)
+      end
+      private_class_method :population_precision
+
+      def varlen8(reader)
+        return 0 if reader.read(1).zero?
+
+        bits = reader.read(3)
+        bits.zero? ? 1 : reader.read(bits) + (1 << bits)
+      end
+      private_class_method :varlen8
+    end
+
+    AliasEntry = Data.define(:cutoff, :right_symbol, :right_offset, :left_frequency, :right_frequency)
+
+    class AliasTable
+      attr_reader :frequencies, :log_alphabet_size
+
+      def initialize(distribution, log_alphabet_size)
+        @frequencies = distribution.dup
+        @log_alphabet_size = log_alphabet_size
+        @entries = build(distribution.dup)
+      end
+
+      def lookup(value)
+        index = value >> entry_log_size
+        position = value & (entry_size - 1)
+        entry = @entries.fetch(index)
+        if position < entry.cutoff
+          [index, position, entry.left_frequency]
+        else
+          [entry.right_symbol, position + entry.right_offset, entry.right_frequency]
+        end
+      end
+
+      def entries = @entries.dup
+
+      private
+
+      def entry_log_size = Histogram::LOG_RANGE - log_alphabet_size
+      def entry_size = 1 << entry_log_size
+
+      # rubocop:disable-next Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      def build(distribution)
+        distribution.pop while distribution.last&.zero?
+        distribution = [Histogram::RANGE] if distribution.empty?
+        table_size = 1 << log_alphabet_size
+        raise CorruptError, "ANS alphabet is too large" if distribution.length > table_size
+        raise CorruptError, "ANS frequencies do not sum to 4096" unless distribution.sum == Histogram::RANGE
+
+        single = distribution.index(Histogram::RANGE)
+        if single
+          return Array.new(table_size) do |index|
+            AliasEntry.new(cutoff: 0, right_symbol: single, right_offset: entry_size * index,
+                           left_frequency: 0, right_frequency: Histogram::RANGE)
+          end
+        end
+
+        cutoffs = distribution + Array.new(table_size - distribution.length, 0)
+        underfull = []
+        overfull = []
+        cutoffs.each_index do |index|
+          (cutoffs[index] > entry_size ? overfull : underfull) << index unless cutoffs[index] == entry_size
+        end
+        right_symbols = Array.new(table_size)
+        right_offsets = Array.new(table_size, 0)
+        until overfull.empty?
+          over = overfull.pop
+          under = underfull.pop or raise CorruptError, "invalid ANS distribution"
+          cutoffs[over] -= entry_size - cutoffs[under]
+          right_symbols[under] = over
+          right_offsets[under] = cutoffs[over]
+          (cutoffs[over] > entry_size ? overfull : underfull) << over unless cutoffs[over] == entry_size
+        end
+
+        Array.new(table_size) do |index|
+          if cutoffs[index] == entry_size
+            right_symbols[index] = index
+            cutoffs[index] = 0
+          else
+            right_offsets[index] -= cutoffs[index]
+          end
+          left = distribution.fetch(index, 0)
+          right = distribution.fetch(right_symbols[index], 0)
+          AliasEntry.new(cutoff: cutoffs[index], right_symbol: right_symbols[index],
+                         right_offset: right_offsets[index], left_frequency: left, right_frequency: right)
+        end
+      end
+    end
+  end
+end

data/lib/jxl/entropy/decoder.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module JXL
+  module Entropy
+    LZ77 = Data.define(:enabled, :min_symbol, :min_length, :length_config, :distance_context)
+
+    class Decoder
+      WINDOW_SIZE = 1 << 20
+      SPECIAL_DISTANCES = [
+        [0, 1], [1, 0], [1, 1], [-1, 1], [0, 2], [2, 0], [1, 2], [-1, 2],
+        [2, 1], [-2, 1], [2, 2], [-2, 2], [0, 3], [3, 0], [1, 3], [-1, 3],
+        [3, 1], [-3, 1], [2, 3], [-2, 3], [3, 2], [-3, 2], [0, 4], [4, 0],
+        [1, 4], [-1, 4], [4, 1], [-4, 1], [3, 3], [-3, 3], [2, 4], [-2, 4],
+        [4, 2], [-4, 2], [0, 5], [3, 4], [-3, 4], [4, 3], [-4, 3], [5, 0],
+        [1, 5], [-1, 5], [5, 1], [-5, 1], [2, 5], [-2, 5], [5, 2], [-5, 2],
+        [4, 4], [-4, 4], [3, 5], [-3, 5], [5, 3], [-5, 3], [0, 6], [6, 0],
+        [1, 6], [-1, 6], [6, 1], [-6, 1], [2, 6], [-2, 6], [6, 2], [-6, 2],
+        [4, 5], [-4, 5], [5, 4], [-5, 4], [3, 6], [-3, 6], [6, 3], [-6, 3],
+        [0, 7], [7, 0], [1, 7], [-1, 7], [5, 5], [-5, 5], [7, 1], [-7, 1],
+        [4, 6], [-4, 6], [6, 4], [-6, 4], [2, 7], [-2, 7], [7, 2], [-7, 2],
+        [3, 7], [-3, 7], [7, 3], [-7, 3], [5, 6], [-5, 6], [6, 5], [-6, 5],
+        [8, 0], [4, 7], [-4, 7], [7, 4], [-7, 4], [8, 1], [8, 2], [6, 6],
+        [-6, 6], [8, 3], [5, 7], [-5, 7], [7, 5], [-7, 5], [8, 4], [6, 7],
+        [-6, 7], [7, 6], [-7, 6], [8, 5], [7, 7], [-7, 7], [8, 6], [8, 7]
+      ].freeze
+
+      attr_reader :context_map
+
+      def self.decode_context_map(reader, count) = read_context_map(reader, count)
+
+      # rubocop:disable-next Metrics/CyclomaticComplexity
+      def self.read(reader, context_count, disallow_lz77: false, distance_multiplier: 0, defer_state: false)
+        raise ResourceLimitError, "too many entropy contexts" unless context_count.between?(1, 65_536)
+
+        lz77 = read_lz77(reader)
+        raise CorruptError, "LZ77 is not allowed here" if lz77.enabled && disallow_lz77
+
+        context_count += 1 if lz77.enabled
+        length_config = lz77.enabled ? HybridUint.read(reader, 8) : HybridUint.new
+        context_map = context_count > 1 ? read_context_map(reader, context_count) : [0]
+        histogram_count = context_map.max + 1
+        prefix = !reader.read(1).zero?
+        log_alphabet_size = prefix ? 15 : reader.read(2) + 5
+        configs = Array.new(histogram_count) { HybridUint.read(reader, log_alphabet_size) }
+        codes = read_codes(reader, histogram_count, log_alphabet_size, prefix)
+        distance_context = context_map.last
+        lz77 = lz77.with(length_config:, distance_context:)
+        new(reader, context_map, configs, codes, prefix, lz77, distance_multiplier, defer_state)
+      end
+
+      # rubocop:disable-next Metrics/ParameterLists
+      def initialize(reader, context_map, configs, codes, prefix, lz77, distance_multiplier, defer_state)
+        @reader = reader
+        @context_map = context_map
+        @configs = configs
+        @codes = codes
+        @prefix = prefix
+        @lz77 = lz77
+        @default_distance_multiplier = distance_multiplier
+        start unless defer_state
+      end
+
+      def start(distance_multiplier: @default_distance_multiplier)
+        raise InternalError, "entropy decoder already started" if @started
+
+        @state = @prefix ? 0x13_0000 : @reader.read(32)
+        @special_distances = if distance_multiplier.zero?
+                               []
+                             else
+                               SPECIAL_DISTANCES.map do |x, y|
+                                 [x + (distance_multiplier * y), 1].max
+                               end
+                             end
+        initialize_window if @lz77.enabled
+        @started = true
+        self
+      end
+
+      def fork(reader, distance_multiplier:)
+        self.class.new(reader, @context_map, @configs, @codes, @prefix, @lz77, distance_multiplier, false)
+      end
+
+      def read_uint(context) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        raise InternalError, "entropy decoder has not started" unless @started
+
+        max_context = context_map.length - (@lz77.enabled ? 2 : 1)
+        raise CorruptError, "invalid entropy context" unless context.between?(0, max_context)
+        return copy_symbol(context) if @copy_remaining&.positive?
+
+        histogram = context_map.fetch(context)
+        token = read_symbol(histogram)
+        Trace.token(histogram, token)
+        if @lz77.enabled && token >= @lz77.min_symbol
+          start_copy(token)
+          return copy_symbol(context)
+        end
+
+        value = @configs.fetch(histogram).decode(token, @reader)
+        store(value) if @lz77.enabled
+        Trace.symbol(context, value)
+        value
+      end
+
+      def final_state!
+        raise CorruptError, "invalid ANS final state (#{@state})" unless @state == 0x13_0000
+
+        true
+      end
+
+      def self.read_context_map(reader, count) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        map = Array.new(count, 0)
+        if reader.read(1).zero?
+          use_mtf = !reader.read(1).zero?
+          nested = read(reader, 1, disallow_lz77: count <= 2)
+          map.map! { nested.read_uint(0) }
+          nested.final_state!
+          inverse_mtf!(map) if use_mtf
+        else
+          bits = reader.read(2)
+          map.map! { reader.read(bits) } unless bits.zero?
+        end
+        raise CorruptError, "context map has too many clusters" if map.max >= 256
+
+        expected = (0..map.max).to_a
+        raise CorruptError, "incomplete context map" unless map.uniq.sort == expected
+
+        map
+      end
+      private_class_method :read_context_map
+
+      def self.inverse_mtf!(map)
+        values = (0..255).to_a
+        map.map! do |index|
+          raise CorruptError, "invalid move-to-front index" if index >= values.length
+
+          value = values.delete_at(index)
+          values.unshift(value)
+          value
+        end
+      end
+      private_class_method :inverse_mtf!
+
+      def self.read_lz77(reader)
+        enabled = !reader.read(1).zero?
+        unless enabled
+          return LZ77.new(enabled: false, min_symbol: 0, min_length: 0, length_config: nil,
+                          distance_context: nil)
+        end
+
+        min_symbol = Bit::Field.u32(reader, Bit::Field.val(224), Bit::Field.val(512), Bit::Field.val(4096),
+                                    Bit::Field.bits_offset(15, 8))
+        min_length = Bit::Field.u32(reader, Bit::Field.val(3), Bit::Field.val(4),
+                                    Bit::Field.bits_offset(2, 5), Bit::Field.bits_offset(8, 9))
+        LZ77.new(enabled: true, min_symbol:, min_length:, length_config: nil, distance_context: nil)
+      end
+      private_class_method :read_lz77
+
+      def self.read_codes(reader, count, log_alphabet_size, prefix)
+        if prefix
+          sizes = Array.new(count) { varlen16(reader) + 1 }
+          limit = 1 << log_alphabet_size
+          raise CorruptError, "prefix alphabet is too large" if sizes.any? { |size| size > limit }
+
+          sizes.map { |size| size == 1 ? PrefixCode.constant(0) : PrefixCode.read(reader, size) }
+        else
+          Array.new(count) { AliasTable.new(Histogram.read(reader), log_alphabet_size) }
+        end
+      end
+      private_class_method :read_codes
+
+      def self.varlen16(reader)
+        return 0 if reader.read(1).zero?
+
+        bits = reader.read(4)
+        bits.zero? ? 1 : reader.read(bits) + (1 << bits)
+      end
+      private_class_method :varlen16
+
+      private
+
+      def read_symbol(histogram)
+        return @codes.fetch(histogram).read(@reader) if @prefix
+
+        residue = @state & 4095
+        symbol, offset, frequency = @codes.fetch(histogram).lookup(residue)
+        @state = (frequency * (@state >> 12)) + offset
+        @state = (@state << 16) | @reader.read(16) if @state < (1 << 16)
+        symbol
+      end
+
+      def initialize_window
+        @window = Array.new(WINDOW_SIZE, 0)
+        @decoded = 0
+        @copy_position = 0
+        @copy_remaining = 0
+      end
+
+      def start_copy(token)
+        @copy_remaining = @lz77.length_config.decode(token - @lz77.min_symbol, @reader) + @lz77.min_length
+        distance_token = read_symbol(@lz77.distance_context)
+        distance = @configs.fetch(@lz77.distance_context).decode(distance_token, @reader)
+        distance = if distance < @special_distances.length
+                     @special_distances[distance]
+                   else
+                     distance + 1 - @special_distances.length
+                   end
+        distance = [distance, @decoded, WINDOW_SIZE].min
+        @copy_position = @decoded - distance
+      end
+
+      def copy_symbol(context)
+        value = @window[@copy_position & (WINDOW_SIZE - 1)]
+        @copy_position += 1
+        @copy_remaining -= 1
+        store(value)
+        Trace.symbol(context, value)
+        value
+      end
+
+      def store(value)
+        @window[@decoded & (WINDOW_SIZE - 1)] = value
+        @decoded += 1
+      end
+    end
+  end
+end

data/lib/jxl/entropy/encoder.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module JXL
+  module Entropy
+    class Encoder
+      def initialize(values)
+        @config = HybridUint.new(split_exponent: 0, msb_in_token: 0, lsb_in_token: 0)
+        @symbols = encode(values)
+        @alphabet_size = alphabet_size
+      end
+
+      def write_header(writer, context_count: 1)
+        writer.write(1, 0) # no LZ77
+        writer.write(1, 1).write(2, 0) if context_count > 1 # one histogram for every context
+        writer.write(1, 1) # prefix coding
+        writer.write(4, 0) # HybridUint split exponent 0
+        write_varlen16(writer, @alphabet_size - 1)
+        write_uniform_prefix_code(writer) if @alphabet_size > 1
+        writer
+      end
+
+      def write_symbols(writer, values = nil)
+        return writer if @alphabet_size == 1
+
+        width = @alphabet_size.bit_length - 1
+        (values ? encode(values) : @symbols).each do |token, extra_count, extra_bits|
+          raise ArgumentError, "symbol exceeds entropy alphabet" if token >= @alphabet_size
+
+          writer.write(width, reverse_bits(token, width))
+          writer.write(extra_count, extra_bits)
+        end
+        writer
+      end
+
+      def write(writer, context_count: 1)
+        write_header(writer, context_count:)
+        write_symbols(writer)
+      end
+
+      private
+
+      def alphabet_size
+        maximum = @symbols.map(&:first).max || 0
+        maximum.zero? ? 1 : 1 << maximum.bit_length
+      end
+
+      def encode(values) = values.map { @config.encode(_1) }
+
+      def write_varlen16(writer, value)
+        return writer.write(1, 0) if value.zero?
+
+        width = value.bit_length - 1
+        writer.write(1, 1).write(4, width).write(width, value - (1 << width))
+      end
+
+      def write_uniform_prefix_code(writer)
+        length = @alphabet_size.bit_length - 1
+        writer.write(2, 0)
+        PrefixCode::CODE_LENGTH_ORDER.each do |symbol|
+          symbol == length ? writer.write(4, 7) : writer.write(2, 0)
+        end
+      end
+
+      def reverse_bits(value, width)
+        width.times.reduce(0) { |result, bit| (result << 1) | value[bit] }
+      end
+    end
+  end
+end

data/lib/jxl/entropy/hybrid_uint.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module JXL
+  module Entropy
+    HybridUint = Data.define(:split_exponent, :msb_in_token, :lsb_in_token) do
+      def initialize(split_exponent: 4, msb_in_token: 2, lsb_in_token: 0)
+        unless split_exponent.between?(0, 15) && msb_in_token >= 0 && lsb_in_token >= 0 &&
+               msb_in_token + lsb_in_token <= split_exponent
+          raise CorruptError, "invalid hybrid integer configuration"
+        end
+
+        super
+      end
+
+      def split_token = 1 << split_exponent
+
+      def decode(token, reader)
+        return token if token < split_token
+
+        token_bits = msb_in_token + lsb_in_token
+        extra_bits = split_exponent - token_bits + ((token - split_token) >> token_bits)
+        extra_bits &= 31
+
+        low = token & ((1 << lsb_in_token) - 1)
+        token >>= lsb_in_token
+        high = (1 << msb_in_token) | (token & ((1 << msb_in_token) - 1))
+        Num.u32((((high << extra_bits) | reader.read(extra_bits)) << lsb_in_token) | low)
+      end
+
+      def encode(value) # rubocop:disable Metrics/AbcSize
+        return [value, 0, 0] if value < split_token
+
+        exponent = value.bit_length - 1
+        mantissa = value - (1 << exponent)
+        token = split_token + ((exponent - split_exponent) << (msb_in_token + lsb_in_token)) +
+                ((mantissa >> (exponent - msb_in_token)) << lsb_in_token) +
+                (mantissa & ((1 << lsb_in_token) - 1))
+        bit_count = exponent - msb_in_token - lsb_in_token
+        bits = (value >> lsb_in_token) & ((1 << bit_count) - 1)
+        [token, bit_count, bits]
+      end
+
+      def self.read(reader, log_alphabet_size)
+        split = reader.read(Num.ceil_log2(log_alphabet_size + 1))
+        return new(split_exponent: split, msb_in_token: 0, lsb_in_token: 0) if split == log_alphabet_size
+
+        msb = reader.read(Num.ceil_log2(split + 1))
+        raise CorruptError, "invalid hybrid integer configuration" if msb > split
+
+        lsb = reader.read(Num.ceil_log2(split - msb + 1))
+        new(split_exponent: split, msb_in_token: msb, lsb_in_token: lsb)
+      end
+    end
+  end
+end