omnizip 0.4.7-x86_64-linux

data/lib/omnizip/algorithms/zstandard/ldm.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Long-Distance Matching sparse hash table (port of the
+      # omnizip-rs encoder/ldm.rs).
+      #
+      # Hashes every `gap`-th position into a head/chain table so the
+      # parser can find matches at distances far beyond the normal
+      # match-finder window — up to the full frame size. The table is
+      # pre-populated over the whole input before parsing starts, so
+      # entries may point forward of the current position; find_match
+      # skips those without spending its chain budget.
+      class LdmHashTable
+        # Chain entries walked per find_match (Rust LDM_MAX_CHAIN).
+        LDM_MAX_CHAIN = 32
+        MIN_MATCH = MatchFinder::MIN_MATCH
+
+        # Hash table and chain sizes are additionally capped by the
+        # number of sparse samples so small inputs do not pay for a
+        # window-sized table.
+        def initialize(input_len, window_log, gap = 16)
+          samples = (input_len / gap) + 1
+          hash_log = [[window_log, 21].min, samples.bit_length + 2].min
+          hash_log = 1 if hash_log < 1
+
+          @gap = gap
+          @hash_log = hash_log
+          @head = Array.new(1 << hash_log)
+          @chain = Array.new(samples)
+        end
+
+        # Insert `pos` into the table (sparse sampling: every gap-th
+        # position only).
+        def insert(src, pos)
+          return unless (pos % @gap).zero?
+
+          h = hash4(src, pos)
+          idx = pos / @gap
+          @chain[idx] = @head[h] if idx < @chain.length
+          @head[h] = pos
+        end
+
+        # Longest match at `pos` within `max_distance`, bounded by
+        # `end_bound` (the current block end). Walks up to MAX_CHAIN
+        # chain entries. Returns [distance, length] or nil.
+        #
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/AbcSize
+        # rubocop:disable-next Metrics/CyclomaticComplexity
+        def find_match(src, pos, max_distance, min_match, end_bound)
+          return nil if pos + MIN_MATCH > src.bytesize
+
+          candidate = @head[hash4(src, pos)]
+          best_len = 0
+          best_dist = 0
+          chain_count = 0
+
+          while candidate && chain_count < LDM_MAX_CHAIN
+            if candidate >= pos
+              candidate = @chain[candidate / @gap]
+              next
+            end
+
+            dist = pos - candidate
+            break if dist > max_distance
+
+            len_cap = [end_bound - pos, pos - candidate].min
+            len_cap = 0 if len_cap.negative?
+            len = MatchFinder.count_match(src, pos, src, candidate, len_cap)
+            if len > best_len && len >= min_match
+              best_len = len
+              best_dist = dist
+            end
+
+            candidate = @chain[candidate / @gap]
+            chain_count += 1
+          end
+
+          return nil unless best_len >= min_match && best_dist.positive?
+
+          [best_dist, best_len]
+        end
+        # rubocop:enable Metrics/AbcSize
+        # rubocop:enable Metrics/MethodLength
+
+        private
+
+        # rubocop:disable-next Metrics/AbcSize
+        def hash4(src, pos)
+          v = src.getbyte(pos) |
+            (src.getbyte(pos + 1) << 8) |
+            (src.getbyte(pos + 2) << 16) |
+            (src.getbyte(pos + 3) << 24)
+          ((v * MatchFinder::PRIME4_BYTES) & 0xFFFFFFFF) >> (32 - @hash_log)
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/literals.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Literals section decoder (RFC 8878 §3.1.1.3.1).
+      #
+      # Header byte 0 layout (bits numbered from the LSB, matching the C
+      # reference `litEncType = istart[0] & 3`):
+      #
+      #   bits 0-1  Literals_Block_Type (0=Raw, 1=RLE, 2=Compressed,
+      #             3=Treeless)
+      #   bits 2-3  Size_Format selector
+      #
+      # Raw / RLE size formats:
+      #   00, 10 -> 1-byte header, regen = byte0 >> 3
+      #   01     -> 2-byte header, regen = LE16 >> 4
+      #   11     -> 3-byte header, regen = LE24 >> 4
+      #
+      # Compressed / Treeless size formats:
+      #   00 -> 3-byte header, single stream, 10-bit sizes
+      #   01 -> 3-byte header, 4 streams, 10-bit sizes
+      #   10 -> 4-byte header, 4 streams, 14-bit sizes
+      #   11 -> 5-byte header, 4 streams, 18-bit sizes
+      class LiteralsDecoder
+        include Constants
+
+        # @return [String] decoded literal bytes
+        attr_reader :literals
+
+        # @return [Huffman, nil] table from a Compressed block, for the
+        #   next Treeless block in the same frame
+        attr_reader :huffman_table
+
+        # @return [Integer] bytes consumed from the input
+        attr_reader :consumed
+
+        # Decode the literals section at the head of `input`.
+        #
+        # @param input [String]
+        # @param previous_table [Huffman, nil] table from the previous
+        #   compressed literals block (required for Treeless)
+        # @return [LiteralsDecoder]
+        def self.decode(input, previous_table = nil)
+          raise Omnizip::DecompressionError, "empty literals section" if input.empty?
+
+          block_type = input.getbyte(0) & 0x03
+
+          case block_type
+          when LITERALS_BLOCK_RAW then decode_raw(input)
+          when LITERALS_BLOCK_RLE then decode_rle(input)
+          when LITERALS_BLOCK_COMPRESSED
+            decode_compressed(input, previous_table, false)
+          when LITERALS_BLOCK_TREELESS
+            decode_compressed(input, previous_table, true)
+          end
+        end
+
+        def self.decode_raw(input)
+          regen_size, header_size = size_format_raw_rle(input)
+          end_pos = header_size + regen_size
+          if input.bytesize < end_pos
+            raise Omnizip::DecompressionError,
+                  "truncated raw literals: need #{end_pos} bytes"
+          end
+
+          new(input.byteslice(header_size, regen_size), nil, end_pos)
+        end
+
+        def self.decode_rle(input)
+          regen_size, header_size = size_format_raw_rle(input)
+          if input.bytesize < header_size + 1
+            raise Omnizip::DecompressionError,
+                  "truncated RLE literals: missing repeated byte"
+          end
+
+          byte = input.getbyte(header_size)
+          new(byte.chr * regen_size, nil, header_size + 1)
+        end
+
+        # rubocop:disable Metrics/AbcSize
+        # rubocop:disable-next Metrics/MethodLength
+        def self.decode_compressed(input, previous_table, is_repeat)
+          if is_repeat && previous_table.nil?
+            raise Omnizip::DecompressionError,
+                  "treeless literals block requires a previous compressed " \
+                  "block in the same frame"
+          end
+
+          lhl_code = (input.getbyte(0) >> 2) & 0x03
+
+          min_header = [3, 3, 4, 5][lhl_code]
+          if input.bytesize < min_header
+            raise Omnizip::DecompressionError,
+                  "truncated compressed literals header: need #{min_header} bytes"
+          end
+
+          lit_size, lit_c_size, lh_size, single_stream =
+            case lhl_code
+            when 0, 1
+              lhc = input.getbyte(0) |
+                (input.getbyte(1) << 8) | (input.getbyte(2) << 16)
+              [(lhc >> 4) & 0x3FF, (lhc >> 14) & 0x3FF, 3, lhl_code.zero?]
+            when 2
+              lhc = input.unpack1("V") & 0xFFFFFFFF
+              [(lhc >> 4) & 0x3FFF, lhc >> 18, 4, false]
+            else
+              low = input.byteslice(0, 4).unpack1("V")
+              high = input.getbyte(4)
+              lhc = low | (high << 32)
+              [(lhc >> 4) & 0x3FFFF, lhc >> 22, 5, false]
+            end
+
+          needed = lh_size + lit_c_size
+          if input.bytesize < needed
+            raise Omnizip::DecompressionError,
+                  "truncated compressed literals: need #{needed} bytes"
+          end
+          compressed = input.byteslice(lh_size, lit_c_size)
+
+          table_bytes = 0
+          table = previous_table
+          unless is_repeat
+            table, table_bytes = HuffmanTableReader.read(compressed)
+          end
+
+          literal_data = compressed.byteslice(table_bytes..)
+          literals = if single_stream
+                       decode_single_stream(table, literal_data, lit_size)
+                     else
+                       decode_four_stream(table, literal_data, lit_size)
+                     end
+
+          new(literals, is_repeat ? nil : table, needed)
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        # rubocop:disable-next Metrics/AbcSize
+        def self.size_format_raw_rle(input)
+          lhl_code = (input.getbyte(0) >> 2) & 0x03
+          header_size = [1, 2, 1, 3][lhl_code]
+          if input.bytesize < header_size
+            raise Omnizip::DecompressionError,
+                  "truncated Raw/RLE literals header: need #{header_size} bytes"
+          end
+
+          case lhl_code
+          when 0, 2
+            [input.getbyte(0) >> 3, 1]
+          when 1
+            [input.unpack1("v") >> 4, 2]
+          else
+            lhc = input.getbyte(0) | (input.getbyte(1) << 8) |
+              (input.getbyte(2) << 16)
+            [lhc >> 4, 3]
+          end
+        end
+
+        # One reverse bitstream decoded into lit_size symbols.
+        def self.decode_single_stream(table, src, lit_size)
+          bs = FSE::BitStream.new(src)
+          out = String.new(encoding: Encoding::BINARY)
+          lit_size.times do
+            out << table.decode(bs).chr
+            bs.reload
+          end
+          out
+        end
+
+        # Jump table + four independent reverse bitstreams, each decoding
+        # ~1/4 of the literals (RFC 8878 §3.1.1.3.1.4).
+        # rubocop:disable Metrics/AbcSize
+        def self.decode_four_stream(table, src, lit_size)
+          if src.bytesize < 10
+            raise Omnizip::DecompressionError,
+                  "4-stream literals too short (need at least 10 bytes)"
+          end
+          # rubocop:enable Metrics/AbcSize
+
+          length1 = src.unpack1("v")
+          length2 = src.byteslice(2, 2).unpack1("v")
+          length3 = src.byteslice(4, 2).unpack1("v")
+          total = src.bytesize - 6
+          if length1 + length2 + length3 > total
+            raise Omnizip::DecompressionError,
+                  "4-stream jump table sizes exceed total stream size"
+          end
+
+          streams = src.byteslice(6..)
+          seg1 = streams.byteslice(0, length1)
+          seg2 = streams.byteslice(length1, length2)
+          seg3 = streams.byteslice(length1 + length2, length3)
+          seg4 = streams.byteslice((length1 + length2 + length3)..)
+
+          segment_size = (lit_size + 3) / 4
+          bounds = [0,
+                    segment_size,
+                    segment_size * 2,
+                    segment_size * 3,
+                    lit_size]
+
+          out = String.new(encoding: Encoding::BINARY)
+          [seg1, seg2, seg3, seg4].each_with_index do |seg, i|
+            bs = FSE::BitStream.new(seg)
+            (bounds[i]...bounds[i + 1]).each do
+              out << table.decode(bs).chr
+              bs.reload
+            end
+          end
+          out
+        end
+
+        def initialize(literals, huffman_table, consumed)
+          @literals = literals
+          @huffman_table = huffman_table
+          @consumed = consumed
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/literals_encoder.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Literals Section encoder (RFC 8878 §3.1.1.3.1): picks Raw, RLE
+      # or Huffman-compressed, whichever is smallest.
+      module LiteralsEncoder
+        include Constants
+
+        module_function
+
+        # Encode a literals section for `literals`.
+        #
+        # @param literals [String]
+        # @return [String] the full section (header + content)
+        def encode(literals)
+          return encode_raw(literals) if literals.bytesize < 16
+
+          return encode_rle(literals) if rle?(literals)
+
+          compressed = nil
+          begin
+            compressed = HuffmanEncoder.encode_literals(literals)
+          rescue Omnizip::CompressionError
+            compressed = nil
+          end
+
+          if compressed && compressed.bytesize < literals.bytesize
+            compressed
+          else
+            encode_raw(literals)
+          end
+        end
+
+        # Raw literals block with the minimal size-format header.
+        def encode_raw(literals)
+          encode_raw_rle_header(LITERALS_BLOCK_RAW, literals.bytesize) +
+            literals
+        end
+
+        # RLE literals block: one byte, repeated.
+        def encode_rle(literals)
+          encode_raw_rle_header(LITERALS_BLOCK_RLE, literals.bytesize) +
+            literals.getbyte(0).chr
+        end
+
+        def rle?(literals)
+          first = literals.getbyte(0)
+          idx = 1
+          while idx < literals.bytesize
+            return false if literals.getbyte(idx) != first
+
+            idx += 1
+          end
+          true
+        end
+
+        # 1/2/3-byte Raw/RLE header per the size-format rules. For the
+        # 2- and 3-byte forms the Size_Format bits (01 / 11) live in
+        # bits 2-3 and the size starts at bit 4.
+        def encode_raw_rle_header(type, size)
+          if size < 32
+            [type | (size << 3)].pack("C")
+          elsif size < 4096
+            lhc = type | 0x04 | (size << 4)
+            [lhc & 0xFF, (lhc >> 8) & 0xFF].pack("CC")
+          else
+            lhc = type | 0x0C | (size << 4)
+            [lhc & 0xFF, (lhc >> 8) & 0xFF, (lhc >> 16) & 0xFF].pack("CCC")
+          end
+        end
+      end
+    end
+  end
+end