omnizip 0.4.7-aarch64-linux

data/lib/omnizip/algorithms/zstandard/fse/table.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      module FSE
+        # One FSE table entry: the symbol decoded in this state and the
+        # transition rule (bits to read, baseline to add). base_val /
+        # nb_add_bits carry the sequence-code meaning of the symbol
+        # (LL/ML baseline + extra bits, or OF base + offset code width)
+        # and are attached by the sequences decoder.
+        State = Struct.new(:symbol, :num_bits, :baseline, :base_val,
+                           :nb_add_bits)
+
+        # FSE decoding table built from a normalized distribution
+        # (RFC 8878 §4.1, "From normalized distribution to decoding
+        # tables"; mirrors the C reference FSE_buildDTable).
+        class Table
+          include Constants
+
+          # @return [Array<State>]
+          attr_reader :states
+
+          # @return [Integer] log2 of the table size
+          attr_reader :accuracy_log
+
+          # @return [Integer] number of symbols in the source alphabet
+          attr_reader :symbol_count
+
+          # Build a table from a distribution.
+          #
+          # @param distribution [Array<Integer>] normalized counts;
+          #   positive = cell count, -1 = "less than 1" low-probability
+          #   symbol (one cell at the top of the table), 0 = absent
+          # @param accuracy_log [Integer]
+          # @return [Table]
+          def self.build(distribution, accuracy_log)
+            table_size = 1 << accuracy_log
+            step = (table_size >> 1) + (table_size >> 3) + 3
+            mask = table_size - 1
+
+            # Phase 1a: low-probability (-1) symbols occupy single cells
+            # from the top of the table downward.
+            table_symbol = Array.new(table_size, 0xFFFF)
+            high_threshold = table_size - 1
+            distribution.each_with_index do |freq, symbol|
+              next unless freq == -1
+
+              table_symbol[high_threshold] = symbol
+              if high_threshold.zero?
+                raise Omnizip::DecompressionError,
+                      "FSE table overflow placing low-probability symbol"
+              end
+
+              high_threshold -= 1
+            end
+
+            # Phase 1b: spread positive-count symbols from position 0,
+            # skipping the reserved low-probability area.
+            position = 0
+            distribution.each_with_index do |freq, symbol|
+              next unless freq.positive?
+
+              freq.times do
+                table_symbol[position] = symbol
+                position = (position + step) & mask
+                position = (position + step) & mask while position > high_threshold
+              end
+            end
+
+            # Phase 2: symbolNext starts at 1 for -1/1 counts, at the
+            # count itself otherwise.
+            symbol_next = Array.new(distribution.length, 0)
+            singular = [-1, 1].freeze
+            distribution.each_with_index do |freq, symbol|
+              next if freq.zero?
+
+              symbol_next[symbol] = singular.include?(freq) ? 1 : freq
+            end
+
+            # Phase 3: per-cell transition rules.
+            states = table_symbol.map do |symbol|
+              next_state = symbol_next[symbol]
+              symbol_next[symbol] += 1
+              nb_bits = accuracy_log - Constants.highbit32(next_state)
+              baseline = (next_state << nb_bits) - table_size
+              State.new(symbol, nb_bits, baseline)
+            end
+
+            new(states, accuracy_log, distribution.length)
+          end
+
+          # Build from an RFC 8878 §4.1.3 predefined distribution.
+          #
+          # @param distribution [Array<Integer>]
+          # @param accuracy_log [Integer]
+          # @return [Table]
+          def self.build_predefined(distribution, accuracy_log)
+            build(distribution, accuracy_log)
+          end
+
+          # Single-symbol RLE table: every state decodes `symbol` with a
+          # full state reset.
+          #
+          # @return [Table]
+          def self.build_rle(symbol, accuracy_log)
+            new(Array.new(1 << accuracy_log) { State.new(symbol, 0, 0) },
+                accuracy_log, 1)
+          end
+
+          def initialize(states, accuracy_log, symbol_count)
+            @states = states
+            @accuracy_log = accuracy_log
+            @symbol_count = symbol_count
+          end
+
+          # @return [State]
+          def [](index)
+            @states[index]
+          end
+
+          # @return [Integer]
+          def size
+            @states.length
+          end
+        end
+
+        # Stateful single-stream FSE decoder.
+        class Decoder
+          # @return [Integer] current state index
+          attr_reader :state
+
+          # @param table [Table]
+          def initialize(table)
+            @table = table
+            @state = 0
+          end
+
+          # Initialize the state by reading accuracy_log bits.
+          def init_state(bitstream)
+            @state = bitstream.read_bits(@table.accuracy_log)
+          end
+
+          # Decode one symbol: read it from the current state, then
+          # transition.
+          #
+          # @return [Integer] symbol
+          def decode(bitstream)
+            entry = @table[@state]
+            @state = if entry.num_bits.positive?
+                       entry.baseline + bitstream.read_bits(entry.num_bits)
+                     else
+                       entry.baseline
+                     end
+            entry.symbol
+          end
+
+          # The symbol the current state decodes, without transitioning.
+          #
+          # @return [Integer]
+          def current_symbol
+            @table[@state].symbol
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/fse/table_description.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      module FSE
+        # Reads an FSE table description ("NCount") from the wire and
+        # builds an FSE decode Table (RFC 8878 §4.1.1, FSE Table
+        # Description; mirrors the C reference FSE_readNCount).
+        #
+        # The distribution is a forward little-endian bit-packed stream:
+        # 4 bits of table log, then per-symbol counts whose bit width
+        # shrinks as probability points are spent, with 2-bit repeat
+        # codes for runs of zero-count symbols.
+        module TableDescription
+          include Constants
+
+          MAX_SYMBOL_VALUE = 255
+
+          # rubocop:disable Metrics/MethodLength
+          # rubocop:disable Metrics/PerceivedComplexity
+          module_function
+
+          # Read a normalized distribution from the head of `src`.
+          #
+          # @param src [String] bytes holding the table description
+          #   (plus whatever follows)
+          # @return [Array(Table, Integer)] the decode table and the
+          #   number of bytes consumed
+          def read(src)
+            raise Omnizip::DecompressionError, "FSE table description truncated" if src.bytesize < 4
+
+            reader = BitReader.new(src)
+
+            table_log = (reader.peek & 0xF) + FSE_MIN_ACCURACY_LOG
+            if table_log > FSE_MAX_ACCURACY_LOG
+              raise Omnizip::DecompressionError,
+                    "FSE tableLog #{table_log} exceeds max #{FSE_MAX_ACCURACY_LOG}"
+            end
+            reader.skip_bits(4)
+
+            remaining = (1 << table_log) + 1
+            threshold = 1 << table_log
+            nb_bits = table_log + 1
+
+            counts = Array.new(MAX_SYMBOL_VALUE + 1, 0)
+            charnum = 0
+            previous0 = false
+
+            loop do
+              if previous0
+                charnum = skip_zero_run(reader, charnum)
+                previous0 = false
+                break if charnum > MAX_SYMBOL_VALUE
+
+                next
+              end
+
+              max = ((2 * threshold) - 1) - remaining
+              masked = reader.peek & (threshold - 1)
+              if masked < max
+                count = masked
+                reader.skip_bits(nb_bits - 1)
+              else
+                count = reader.peek & ((2 * threshold) - 1)
+                count -= max if count >= threshold
+                reader.skip_bits(nb_bits)
+              end
+
+              actual = count - 1
+              actual.negative? ? remaining += actual : remaining -= actual
+              counts[charnum] = actual
+              charnum += 1
+              previous0 = actual.zero?
+
+              if remaining < threshold
+                break if remaining <= 1
+
+                high = Constants.highbit32(remaining)
+                nb_bits = high + 1
+                threshold = 1 << high
+              end
+              break if charnum > MAX_SYMBOL_VALUE
+            end
+
+            if remaining != 1
+              raise Omnizip::DecompressionError,
+                    "FSE distribution does not sum to table size (remaining=#{remaining})"
+            end
+            if charnum.zero?
+              raise Omnizip::DecompressionError, "FSE distribution is empty"
+            end
+
+            counts = counts.first(charnum)
+            [Table.build(counts, table_log), reader.bytes_consumed]
+          end
+          # rubocop:enable Metrics/PerceivedComplexity
+          # rubocop:enable Metrics/MethodLength
+
+          # Count 2-bit repeat codes after a zero-count symbol.
+          def skip_zero_run(reader, charnum)
+            inverted = (~reader.peek) | 0x80000000
+            repeats = trailing_zeros(inverted) >> 1
+            while repeats >= 12
+              charnum += 36
+              charnum = MAX_SYMBOL_VALUE + 1 if charnum > MAX_SYMBOL_VALUE + 1
+              reader.skip_bits(24)
+              inverted = (~reader.peek) | 0x80000000
+              repeats = trailing_zeros(inverted) >> 1
+            end
+            charnum += 3 * repeats
+            charnum = MAX_SYMBOL_VALUE + 1 if charnum > MAX_SYMBOL_VALUE + 1
+            reader.skip_bits(2 * repeats)
+
+            tail = reader.peek & 3
+            charnum += tail
+            charnum = MAX_SYMBOL_VALUE + 1 if charnum > MAX_SYMBOL_VALUE + 1
+            reader.skip_bits(2)
+
+            charnum
+          end
+
+          def trailing_zeros(value)
+            count = 0
+            while value.nobits?(1) && count < 32
+              value >>= 1
+              count += 1
+            end
+            count
+          end
+
+          # Forward little-endian bit reader over a byte string with a
+          # sliding 32-bit window (C FSE_readNCount's bit reader).
+          class BitReader
+            def initialize(src)
+              @src = src
+              @byte_pos = 0
+              @bit_count = 0
+              @bit_window = read_u32(0)
+            end
+
+            def peek
+              @bit_window
+            end
+
+            def skip_bits(bits)
+              refill(bits)
+            end
+
+            def bytes_consumed
+              @byte_pos + ((@bit_count + 7) >> 3)
+            end
+
+            private
+
+            def refill(extra_bits)
+              @bit_count += extra_bits
+              advance = @bit_count >> 3
+              if @byte_pos + advance + 4 > @src.bytesize
+                # Clamp to the last 4 bytes, as the C reference does.
+                new_pos = @src.bytesize - 4
+                consumed = (new_pos - @byte_pos) * 8
+                @bit_count = (((@bit_count - consumed) % 32) + 32) % 32
+                @byte_pos = new_pos
+              else
+                @byte_pos += advance
+                @bit_count &= 7
+              end
+              @bit_window = read_u32(@byte_pos) >> @bit_count
+            end
+
+            def read_u32(offset)
+              @src.getbyte(offset) |
+                (@src.getbyte(offset + 1) << 8) |
+                (@src.getbyte(offset + 2) << 16) |
+                (@src.getbyte(offset + 3) << 24)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/fse.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # FSE (Finite State Entropy) namespace (RFC 8878 §4.1).
+      module FSE
+        autoload :BitStream, "omnizip/algorithms/zstandard/fse/bitstream"
+        autoload :Table, "omnizip/algorithms/zstandard/fse/table"
+        autoload :TableDescription,
+                 "omnizip/algorithms/zstandard/fse/table_description"
+        autoload :Interleaved, "omnizip/algorithms/zstandard/fse/interleaved"
+        autoload :Encoder, "omnizip/algorithms/zstandard/fse/encoder"
+
+        module_function
+
+        # Read an FSE table description from the head of `src`.
+        #
+        # @return [Array(Table, Integer)] table and bytes consumed
+        def read_table(src)
+          TableDescription.read(src)
+        end
+
+        # Decode a 2-state interleaved FSE stream.
+        #
+        # @return [Array<Integer>]
+        def decode_stream(table, bitstream_bytes, max_output)
+          Interleaved.decode_stream(table, bitstream_bytes, max_output)
+        end
+      end
+    end
+  end
+end