omnizip 0.3.13 → 0.3.15

data/lib/omnizip/algorithms/zstandard/fse/interleaved.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      module FSE
+        # 2-state interleaved FSE stream decoder (RFC 8878 §4.2.1.2,
+        # "FSE compression of Huffman weights"; mirrors the C reference
+        # FSE_decompress_usingDTable).
+        #
+        # Symbols are produced until the bitstream overflows; after an
+        # overflow following one state's transition, one final symbol is
+        # decoded from the other state without transitioning.
+        module Interleaved
+          module_function
+
+          # Decode a 2-state interleaved FSE stream.
+          #
+          # @param table [Table]
+          # @param bitstream_bytes [String]
+          # @param max_output [Integer] cap on decoded length (corrupt
+          #   input protection)
+          # @return [Array<Integer>] decoded symbols
+          def decode_stream(table, bitstream_bytes, max_output)
+            bs = BitStream.new(bitstream_bytes)
+            accuracy_log = table.accuracy_log
+
+            s1 = bs.read_bits(accuracy_log)
+            bs.reload
+            s2 = bs.read_bits(accuracy_log)
+            bs.reload
+
+            out = []
+
+            loop do
+              check_length!(out, max_output)
+              e1 = table[s1]
+              out << e1.symbol
+              s1 = e1.baseline + bs.read_bits(e1.num_bits)
+
+              if bs.reload_status == BitStream::OVERFLOW
+                check_length!(out, max_output)
+                out << table[s2].symbol
+                break
+              end
+
+              check_length!(out, max_output)
+              e2 = table[s2]
+              out << e2.symbol
+              s2 = e2.baseline + bs.read_bits(e2.num_bits)
+
+              if bs.reload_status == BitStream::OVERFLOW
+                check_length!(out, max_output)
+                out << table[s1].symbol
+                break
+              end
+            end
+
+            out
+          end
+
+          def check_length!(out, max_output)
+            return unless out.length >= max_output
+
+            raise Omnizip::DecompressionError,
+                  "FSE decode exceeded max output #{max_output}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/fse/table.rb

@@ -24,240 +24,162 @@ module Omnizip
   module Algorithms
     class Zstandard
       module FSE
-        # FSE state entry for decoding table
-        #
-        # Each entry contains:
-        # - symbol: The symbol this state decodes to
-        # - num_bits: Number of bits to read for next state
-        # - baseline: Value to add to next state's value
-        State = Struct.new(:symbol, :num_bits, :baseline)
-
-        # FSE decoding table (RFC 8878 Section 4.1)
-        #
-        # Builds a decoding table from a probability distribution
-        # according to RFC 8878.
+        # One FSE table entry: the symbol decoded in this state and the
+        # transition rule (bits to read, baseline to add). base_val /
+        # nb_add_bits carry the sequence-code meaning of the symbol
+        # (LL/ML baseline + extra bits, or OF base + offset code width)
+        # and are attached by the sequences decoder.
+        State = Struct.new(:symbol, :num_bits, :baseline, :base_val,
+                           :nb_add_bits)
+
+        # FSE decoding table built from a normalized distribution
+        # (RFC 8878 §4.1, "From normalized distribution to decoding
+        # tables"; mirrors the C reference FSE_buildDTable).
         class Table
           include Constants
 
-          # @return [Array<State>] Decoding table entries
+          # @return [Array<State>]
           attr_reader :states
 
-          # @return [Integer] Accuracy log (table size = 2^accuracy_log)
+          # @return [Integer] log2 of the table size
           attr_reader :accuracy_log
 
-          # @return [Integer] Number of symbols in the table
+          # @return [Integer] number of symbols in the source alphabet
           attr_reader :symbol_count
 
-          # Build FSE table from normalized distribution
+          # Build a table from a distribution.
           #
-          # @param distribution [Array<Integer>] Normalized symbol frequencies
-          # @param accuracy_log [Integer] Log2 of table size
-          # @return [Table] Built FSE table
+          # @param distribution [Array<Integer>] normalized counts;
+          #   positive = cell count, -1 = "less than 1" low-probability
+          #   symbol (one cell at the top of the table), 0 = absent
+          # @param accuracy_log [Integer]
+          # @return [Table]
           def self.build(distribution, accuracy_log)
             table_size = 1 << accuracy_log
+            step = (table_size >> 1) + (table_size >> 3) + 3
+            mask = table_size - 1
+
+            # Phase 1a: low-probability (-1) symbols occupy single cells
+            # from the top of the table downward.
+            table_symbol = Array.new(table_size, 0xFFFF)
+            high_threshold = table_size - 1
+            distribution.each_with_index do |freq, symbol|
+              next unless freq == -1
+
+              table_symbol[high_threshold] = symbol
+              if high_threshold.zero?
+                raise Omnizip::DecompressionError,
+                      "FSE table overflow placing low-probability symbol"
+              end
+
+              high_threshold -= 1
+            end
 
-            # Allocate cells using spread pattern
-            cells = allocate_cells(distribution, table_size)
+            # Phase 1b: spread positive-count symbols from position 0,
+            # skipping the reserved low-probability area.
+            position = 0
+            distribution.each_with_index do |freq, symbol|
+              next unless freq.positive?
+
+              freq.times do
+                table_symbol[position] = symbol
+                position = (position + step) & mask
+                position = (position + step) & mask while position > high_threshold
+              end
+            end
 
-            # Calculate num_bits and baseline for each state
-            states = calculate_state_values(cells, distribution, table_size)
+            # Phase 2: symbolNext starts at 1 for -1/1 counts, at the
+            # count itself otherwise.
+            symbol_next = Array.new(distribution.length, 0)
+            singular = [-1, 1].freeze
+            distribution.each_with_index do |freq, symbol|
+              next if freq.zero?
+
+              symbol_next[symbol] = singular.include?(freq) ? 1 : freq
+            end
+
+            # Phase 3: per-cell transition rules.
+            states = table_symbol.map do |symbol|
+              next_state = symbol_next[symbol]
+              symbol_next[symbol] += 1
+              nb_bits = accuracy_log - Constants.highbit32(next_state)
+              baseline = (next_state << nb_bits) - table_size
+              State.new(symbol, nb_bits, baseline)
+            end
 
             new(states, accuracy_log, distribution.length)
           end
 
-          # Build from predefined distribution
+          # Build from an RFC 8878 §4.1.3 predefined distribution.
           #
-          # @param distribution [Array<Integer>] Predefined distribution
-          # @param accuracy_log [Integer] Accuracy log
+          # @param distribution [Array<Integer>]
+          # @param accuracy_log [Integer]
           # @return [Table]
           def self.build_predefined(distribution, accuracy_log)
             build(distribution, accuracy_log)
           end
 
-          # Initialize with pre-built table
+          # Single-symbol RLE table: every state decodes `symbol` with a
+          # full state reset.
           #
-          # @param states [Array<State>] Decoding states
-          # @param accuracy_log [Integer]
-          # @param symbol_count [Integer]
+          # @return [Table]
+          def self.build_rle(symbol, accuracy_log)
+            new(Array.new(1 << accuracy_log) { State.new(symbol, 0, 0) },
+                accuracy_log, 1)
+          end
+
           def initialize(states, accuracy_log, symbol_count)
             @states = states
             @accuracy_log = accuracy_log
             @symbol_count = symbol_count
           end
 
-          # Get state at index
-          #
-          # @param index [Integer] State index
-          # @return [State] State at index
+          # @return [State]
           def [](index)
             @states[index]
           end
 
-          # Get table size
-          #
-          # @return [Integer] Number of entries in table
+          # @return [Integer]
           def size
             @states.length
           end
-
-          # Allocate cells using FSE spread pattern
-          #
-          # The spread pattern distributes symbols across the table
-          # using a step that ensures good distribution.
-          def self.allocate_cells(distribution, table_size)
-            cells = Array.new(table_size, nil)
-
-            # Validate distribution sum
-            total = distribution.compact.sum
-            if total > table_size
-              raise ArgumentError,
-                    "Distribution sum (#{total}) exceeds table size (#{table_size})"
-            end
-
-            # Step = (table_size >> 1) + (table_size >> 3) + 3
-            step = (table_size >> 1) + (table_size >> 3) + 3
-            mask = table_size - 1
-
-            position = 0
-
-            distribution.each_with_index do |prob, symbol|
-              next if prob.nil? || prob <= 0
-
-              prob.times do
-                # Find empty position (with safety limit)
-                attempts = 0
-                while cells[position]
-                  position = (position + step) & mask
-                  attempts += 1
-                  if attempts > table_size
-                    raise "FSE table allocation failed: no empty cell found"
-                  end
-                end
-
-                cells[position] = symbol
-                position = (position + step) & mask
-              end
-            end
-
-            cells
-          end
-
-          # Calculate num_bits and baseline for each state
-          def self.calculate_state_values(cells, distribution, table_size)
-            states = Array.new(table_size)
-
-            # Group positions by symbol
-            symbol_positions = {}
-            cells.each_with_index do |symbol, pos|
-              next if symbol.nil?
-
-              symbol_positions[symbol] ||= []
-              symbol_positions[symbol] << pos
-            end
-
-            # Calculate state values for each symbol
-            symbol_positions.each do |symbol, positions|
-              prob = distribution[symbol]
-              next if prob.nil? || prob <= 0
-
-              positions.each_with_index do |pos, idx|
-                # Calculate num_bits: -log2(prob/table_size)
-                num_bits = calculate_num_bits(prob, table_size)
-
-                # Calculate baseline
-                baseline = idx
-
-                states[pos] = State.new(symbol, num_bits, baseline)
-              end
-            end
-
-            states
-          end
-
-          # Calculate number of bits needed for a symbol with given probability
-          def self.calculate_num_bits(prob, table_size)
-            return 0 if prob <= 0
-
-            # num_bits = accuracy_log - log2(prob)
-            # This is the number of extra bits needed
-            log_prob = 0
-            temp = prob
-            while temp > 1
-              log_prob += 1
-              temp >>= 1
-            end
-
-            log_table = 0
-            temp = table_size
-            while temp > 1
-              log_table += 1
-              temp >>= 1
-            end
-
-            [0, log_table - log_prob].max
-          end
         end
 
-        # FSE Decoder (RFC 8878 Section 4.1)
-        #
-        # Decodes symbols from FSE-encoded bitstreams.
+        # Stateful single-stream FSE decoder.
         class Decoder
-          # @return [Table] FSE decoding table
-          attr_reader :table
-
-          # @return [Integer] Current state
+          # @return [Integer] current state index
           attr_reader :state
 
-          # Initialize decoder with FSE table
-          #
-          # @param table [Table] FSE decoding table
+          # @param table [Table]
           def initialize(table)
             @table = table
             @state = 0
           end
 
-          # Initialize state from bitstream
-          #
-          # @param bitstream [BitStream] The bitstream to read from
+          # Initialize the state by reading accuracy_log bits.
           def init_state(bitstream)
             @state = bitstream.read_bits(@table.accuracy_log)
           end
 
-          # Decode next symbol from bitstream
+          # Decode one symbol: read it from the current state, then
+          # transition.
           #
-          # @param bitstream [BitStream] The bitstream to read from
-          # @return [Integer] Decoded symbol
+          # @return [Integer] symbol
           def decode(bitstream)
             entry = @table[@state]
-            return 0 if entry.nil?
-
-            symbol = entry.symbol
-
-            # Read extra bits for next state
-            if entry.num_bits.positive?
-              extra = bitstream.read_bits(entry.num_bits)
-              @state = entry.baseline + extra
-            else
-              @state = entry.baseline
-            end
-
-            # Mask state to table size
-            @state &= (@table.size - 1)
-
-            symbol
+            @state = if entry.num_bits.positive?
+                       entry.baseline + bitstream.read_bits(entry.num_bits)
+                     else
+                       entry.baseline
+                     end
+            entry.symbol
           end
 
-          # Decode multiple symbols
+          # The symbol the current state decodes, without transitioning.
           #
-          # @param bitstream [BitStream] The bitstream to read from
-          # @param count [Integer] Number of symbols to decode
-          # @return [Array<Integer>] Decoded symbols
-          def decode_symbols(bitstream, count)
-            symbols = []
-            count.times do
-              symbols << decode(bitstream)
-            end
-            symbols
+          # @return [Integer]
+          def current_symbol
+            @table[@state].symbol
           end
         end
       end

data/lib/omnizip/algorithms/zstandard/fse/table_description.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      module FSE
+        # Reads an FSE table description ("NCount") from the wire and
+        # builds an FSE decode Table (RFC 8878 §4.1.1, FSE Table
+        # Description; mirrors the C reference FSE_readNCount).
+        #
+        # The distribution is a forward little-endian bit-packed stream:
+        # 4 bits of table log, then per-symbol counts whose bit width
+        # shrinks as probability points are spent, with 2-bit repeat
+        # codes for runs of zero-count symbols.
+        module TableDescription
+          include Constants
+
+          MAX_SYMBOL_VALUE = 255
+
+          # rubocop:disable Metrics/MethodLength
+          # rubocop:disable Metrics/PerceivedComplexity
+          module_function
+
+          # Read a normalized distribution from the head of `src`.
+          #
+          # @param src [String] bytes holding the table description
+          #   (plus whatever follows)
+          # @return [Array(Table, Integer)] the decode table and the
+          #   number of bytes consumed
+          def read(src)
+            raise Omnizip::DecompressionError, "FSE table description truncated" if src.bytesize < 4
+
+            reader = BitReader.new(src)
+
+            table_log = (reader.peek & 0xF) + FSE_MIN_ACCURACY_LOG
+            if table_log > FSE_MAX_ACCURACY_LOG
+              raise Omnizip::DecompressionError,
+                    "FSE tableLog #{table_log} exceeds max #{FSE_MAX_ACCURACY_LOG}"
+            end
+            reader.skip_bits(4)
+
+            remaining = (1 << table_log) + 1
+            threshold = 1 << table_log
+            nb_bits = table_log + 1
+
+            counts = Array.new(MAX_SYMBOL_VALUE + 1, 0)
+            charnum = 0
+            previous0 = false
+
+            loop do
+              if previous0
+                charnum = skip_zero_run(reader, charnum)
+                previous0 = false
+                break if charnum > MAX_SYMBOL_VALUE
+
+                next
+              end
+
+              max = ((2 * threshold) - 1) - remaining
+              masked = reader.peek & (threshold - 1)
+              if masked < max
+                count = masked
+                reader.skip_bits(nb_bits - 1)
+              else
+                count = reader.peek & ((2 * threshold) - 1)
+                count -= max if count >= threshold
+                reader.skip_bits(nb_bits)
+              end
+
+              actual = count - 1
+              actual.negative? ? remaining += actual : remaining -= actual
+              counts[charnum] = actual
+              charnum += 1
+              previous0 = actual.zero?
+
+              if remaining < threshold
+                break if remaining <= 1
+
+                high = Constants.highbit32(remaining)
+                nb_bits = high + 1
+                threshold = 1 << high
+              end
+              break if charnum > MAX_SYMBOL_VALUE
+            end
+
+            if remaining != 1
+              raise Omnizip::DecompressionError,
+                    "FSE distribution does not sum to table size (remaining=#{remaining})"
+            end
+            if charnum.zero?
+              raise Omnizip::DecompressionError, "FSE distribution is empty"
+            end
+
+            counts = counts.first(charnum)
+            [Table.build(counts, table_log), reader.bytes_consumed]
+          end
+          # rubocop:enable Metrics/PerceivedComplexity
+          # rubocop:enable Metrics/MethodLength
+
+          # Count 2-bit repeat codes after a zero-count symbol.
+          def skip_zero_run(reader, charnum)
+            inverted = (~reader.peek) | 0x80000000
+            repeats = trailing_zeros(inverted) >> 1
+            while repeats >= 12
+              charnum += 36
+              charnum = MAX_SYMBOL_VALUE + 1 if charnum > MAX_SYMBOL_VALUE + 1
+              reader.skip_bits(24)
+              inverted = (~reader.peek) | 0x80000000
+              repeats = trailing_zeros(inverted) >> 1
+            end
+            charnum += 3 * repeats
+            charnum = MAX_SYMBOL_VALUE + 1 if charnum > MAX_SYMBOL_VALUE + 1
+            reader.skip_bits(2 * repeats)
+
+            tail = reader.peek & 3
+            charnum += tail
+            charnum = MAX_SYMBOL_VALUE + 1 if charnum > MAX_SYMBOL_VALUE + 1
+            reader.skip_bits(2)
+
+            charnum
+          end
+
+          def trailing_zeros(value)
+            count = 0
+            while value.nobits?(1) && count < 32
+              value >>= 1
+              count += 1
+            end
+            count
+          end
+
+          # Forward little-endian bit reader over a byte string with a
+          # sliding 32-bit window (C FSE_readNCount's bit reader).
+          class BitReader
+            def initialize(src)
+              @src = src
+              @byte_pos = 0
+              @bit_count = 0
+              @bit_window = read_u32(0)
+            end
+
+            def peek
+              @bit_window
+            end
+
+            def skip_bits(bits)
+              refill(bits)
+            end
+
+            def bytes_consumed
+              @byte_pos + ((@bit_count + 7) >> 3)
+            end
+
+            private
+
+            def refill(extra_bits)
+              @bit_count += extra_bits
+              advance = @bit_count >> 3
+              if @byte_pos + advance + 4 > @src.bytesize
+                # Clamp to the last 4 bytes, as the C reference does.
+                new_pos = @src.bytesize - 4
+                consumed = (new_pos - @byte_pos) * 8
+                @bit_count = (((@bit_count - consumed) % 32) + 32) % 32
+                @byte_pos = new_pos
+              else
+                @byte_pos += advance
+                @bit_count &= 7
+              end
+              @bit_window = read_u32(@byte_pos) >> @bit_count
+            end
+
+            def read_u32(offset)
+              @src.getbyte(offset) |
+                (@src.getbyte(offset + 1) << 8) |
+                (@src.getbyte(offset + 2) << 16) |
+                (@src.getbyte(offset + 3) << 24)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/fse.rb

@@ -23,11 +23,30 @@
 module Omnizip
   module Algorithms
     class Zstandard
-      # FSE (Finite State Entropy) module namespace
+      # FSE (Finite State Entropy) namespace (RFC 8878 §4.1).
       module FSE
         autoload :BitStream, "omnizip/algorithms/zstandard/fse/bitstream"
-        autoload :Encoder, "omnizip/algorithms/zstandard/fse/encoder"
         autoload :Table, "omnizip/algorithms/zstandard/fse/table"
+        autoload :TableDescription,
+                 "omnizip/algorithms/zstandard/fse/table_description"
+        autoload :Interleaved, "omnizip/algorithms/zstandard/fse/interleaved"
+        autoload :Encoder, "omnizip/algorithms/zstandard/fse/encoder"
+
+        module_function
+
+        # Read an FSE table description from the head of `src`.
+        #
+        # @return [Array(Table, Integer)] table and bytes consumed
+        def read_table(src)
+          TableDescription.read(src)
+        end
+
+        # Decode a 2-state interleaved FSE stream.
+        #
+        # @return [Array<Integer>]
+        def decode_stream(table, bitstream_bytes, max_output)
+          Interleaved.decode_stream(table, bitstream_bytes, max_output)
+        end
       end
     end
   end