omnizip 0.3.17 → 0.3.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26bb587688afe9104a60ed3db30bd0b4d4a0ad909d1f02d0afaf3c8b67343119
-  data.tar.gz: 674f6bdf6fa36ec488ad06aa3f0a98ba792945fff3d244e9913ee06ff64402a6
+  metadata.gz: affda0906d52a418254670adb02e96312d6e268d1df4aafbba575ade503edfbe
+  data.tar.gz: ae4e84af21221350bf4d67a1b295546674527e091b500a715cb96f8547870990
 SHA512:
-  metadata.gz: ed59928f26cec56627b12304d9cf55d3ed04a7ee07f0a3bf2ef6f71b44a0e21d76d4246746db8f5dd0751139f68208a5a5a53c81ce9959f4433f883d3648adfc
-  data.tar.gz: 311d21ef3715257f1db58f9c1238200d04dce604022337fa53adf7a2c95d1a2d1272ba75bbcae55aaaf5d5b97a16ef954e7ab1b423e9f6875abb1f1036653a6b
+  metadata.gz: 91044c5b1067662c5177e2b8ca262ad0434e4a4cf94e9ea321251723499188fc4b57b3793e2ed88ce51ea406cdb1bfb6d31921ef3b08a7baaf4fd6e62102dc5b
+  data.tar.gz: b6614848a2eb450b748bca1d3d297b7bcd5751607124687237c75ebdeeb43b033ffe6d9af49af457e94aa1cace5ab615c3fe6d04f6a5bb3206a863249225ead6

data/lib/omnizip/algorithms/zstandard/constants.rb

@@ -85,6 +85,10 @@ module Omnizip
         MAX_LEVEL = 22
         DEFAULT_LEVEL = 3
 
+        # Long-distance matching kicks in at this level for inputs
+        # larger than one block (Btultra2 in the C strategy table).
+        LDM_MIN_LEVEL = 19
+
         # Buffer size for streaming operations
         BUFFER_SIZE = 128 * 1024 # 128KB
 

data/lib/omnizip/algorithms/zstandard/encoder.rb

@@ -95,6 +95,21 @@ module Omnizip
           ms = MatchFinder::MatchState.new(params[:hash_log])
           ms.enable_chain(params[:chain]) if params[:chain].positive?
 
+          # LDM (long-distance matching) at high levels on multi-block
+          # inputs: a sparse table over the whole frame finds matches
+          # beyond the 128 KiB block window. Chains are disabled —
+          # the chain table is block-sized and LDM provides the
+          # long-distance coverage instead (Rust reference behavior).
+          ldm = nil
+          max_distance = BLOCK_MAX_SIZE
+          if @level >= LDM_MIN_LEVEL && data.bytesize > BLOCK_MAX_SIZE
+            window_log = data.bytesize.bit_length.clamp(10, 31)
+            ldm = LdmHashTable.new(data.bytesize, window_log)
+            (0...data.bytesize).each { |pos| ldm.insert(data, pos) }
+            ms.disable_chain
+            max_distance = data.bytesize
+          end
+
           # Wire repeat-offset state carried across blocks; matches the
           # decoder's executor state. Raw and RLE blocks leave it
           # untouched.
@@ -114,7 +129,7 @@ module Omnizip
               seq_store = MatchFinder::SeqStore.new(reps.dup)
               MatchFinder.compress_range(data, offset, block_end, seq_store,
                                          ms, params[:min_match],
-                                         params[:lazy])
+                                         params[:lazy], ldm, max_distance)
               content, new_reps = try_compressed(seq_store, reps)
               if content.nil?
                 write_block_header(is_last ? 1 : 0, BLOCK_TYPE_RAW,

data/lib/omnizip/algorithms/zstandard/ldm.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Long-Distance Matching sparse hash table (port of the
+      # omnizip-rs encoder/ldm.rs).
+      #
+      # Hashes every `gap`-th position into a head/chain table so the
+      # parser can find matches at distances far beyond the normal
+      # match-finder window — up to the full frame size. The table is
+      # pre-populated over the whole input before parsing starts, so
+      # entries may point forward of the current position; find_match
+      # skips those without spending its chain budget.
+      class LdmHashTable
+        # Chain entries walked per find_match (Rust LDM_MAX_CHAIN).
+        LDM_MAX_CHAIN = 32
+        MIN_MATCH = MatchFinder::MIN_MATCH
+
+        # Hash table and chain sizes are additionally capped by the
+        # number of sparse samples so small inputs do not pay for a
+        # window-sized table.
+        def initialize(input_len, window_log, gap = 16)
+          samples = (input_len / gap) + 1
+          hash_log = [[window_log, 21].min, samples.bit_length + 2].min
+          hash_log = 1 if hash_log < 1
+
+          @gap = gap
+          @hash_log = hash_log
+          @head = Array.new(1 << hash_log)
+          @chain = Array.new(samples)
+        end
+
+        # Insert `pos` into the table (sparse sampling: every gap-th
+        # position only).
+        def insert(src, pos)
+          return unless (pos % @gap).zero?
+
+          h = hash4(src, pos)
+          idx = pos / @gap
+          @chain[idx] = @head[h] if idx < @chain.length
+          @head[h] = pos
+        end
+
+        # Longest match at `pos` within `max_distance`, bounded by
+        # `end_bound` (the current block end). Walks up to MAX_CHAIN
+        # chain entries. Returns [distance, length] or nil.
+        #
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/AbcSize
+        # rubocop:disable-next Metrics/CyclomaticComplexity
+        def find_match(src, pos, max_distance, min_match, end_bound)
+          return nil if pos + MIN_MATCH > src.bytesize
+
+          candidate = @head[hash4(src, pos)]
+          best_len = 0
+          best_dist = 0
+          chain_count = 0
+
+          while candidate && chain_count < LDM_MAX_CHAIN
+            if candidate >= pos
+              candidate = @chain[candidate / @gap]
+              next
+            end
+
+            dist = pos - candidate
+            break if dist > max_distance
+
+            len_cap = [end_bound - pos, pos - candidate].min
+            len_cap = 0 if len_cap.negative?
+            len = MatchFinder.count_match(src, pos, src, candidate, len_cap)
+            if len > best_len && len >= min_match
+              best_len = len
+              best_dist = dist
+            end
+
+            candidate = @chain[candidate / @gap]
+            chain_count += 1
+          end
+
+          return nil unless best_len >= min_match && best_dist.positive?
+
+          [best_dist, best_len]
+        end
+        # rubocop:enable Metrics/AbcSize
+        # rubocop:enable Metrics/MethodLength
+
+        private
+
+        # rubocop:disable-next Metrics/AbcSize
+        def hash4(src, pos)
+          v = src.getbyte(pos) |
+            (src.getbyte(pos + 1) << 8) |
+            (src.getbyte(pos + 2) << 16) |
+            (src.getbyte(pos + 3) << 24)
+          ((v * MatchFinder::PRIME4_BYTES) & 0xFFFFFFFF) >> (32 - @hash_log)
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/match_finder.rb

@@ -163,7 +163,8 @@ module Omnizip
         # rubocop:disable Metrics/CyclomaticComplexity
         # rubocop:disable-next Metrics/PerceivedComplexity
         def compress_range(src, block_start, block_end, seq_store, ms,
-                           min_match, lazy)
+                           min_match, lazy, ldm = nil,
+                           max_distance = BLOCK_MAX_SIZE)
           mm = [min_match, MIN_MATCH_ECONOMICAL].max
           span = block_end - block_start
           if span < mm + 1
@@ -177,7 +178,11 @@ module Omnizip
           limit = block_end - mm
 
           while ip < limit
-            match = if lazy.zero?
+            match = if ldm
+                      find_match_ldm(src, ip, ms, mm, limit, anchor,
+                                     seq_store.rep_offsets[0], ldm,
+                                     max_distance)
+                    elsif lazy.zero?
                       find_greedy_match(src, ip, ms, mm, limit, anchor,
                                         seq_store.rep_offsets[0])
                     else
@@ -190,7 +195,12 @@ module Omnizip
                 (1..lazy).each do |k|
                   next unless ip + k < limit
 
-                  m2 = probe_match(src, ip + k, ms, mm, limit)
+                  m2 = if ldm
+                         probe_match(src, ip + k, ms, mm, limit, ldm,
+                                     max_distance)
+                       else
+                         probe_match(src, ip + k, ms, mm, limit)
+                       end
                   if m2 && m2[1] > len + k
                     defer = true
                     break
@@ -224,6 +234,22 @@ module Omnizip
         # rubocop:enable Metrics/AbcSize
         # rubocop:enable Metrics/MethodLength
 
+        # LDM-mode match search (port of the Rust
+        # compress_block_lazy2_with_ldm loop body): rep0 fast-path
+        # first, then the normal hash probe merged with the LDM
+        # table. Returns [offset, length, start] or nil.
+        def find_match_ldm(src, ip, ms, min_match, limit, anchor, rep0,
+                           ldm, max_distance)
+          m = rep0_match(src, ip, rep0, min_match, limit, anchor)
+          return m if m
+
+          dist, len = find_best_match(src, ip, ms, min_match, limit,
+                                      ldm, max_distance)
+          return nil unless dist
+
+          [dist, len, ip]
+        end
+
         # Greedy-path match search (port of the Rust
         # compress_block_with_min_match): the rep0 fast-path first
         # (cheapest offset to encode), then the hash table — both
@@ -280,7 +306,8 @@ module Omnizip
         # rubocop:disable Metrics/MethodLength
         # rubocop:disable Metrics/AbcSize
         # rubocop:disable Metrics/CyclomaticComplexity
-        def find_best_match(src, ip, ms, min_match, limit)
+        def find_best_match(src, ip, ms, min_match, limit, ldm = nil,
+                            max_distance = BLOCK_MAX_SIZE)
           size = src.bytesize
           return nil if ip + MIN_MATCH > size
 
@@ -300,7 +327,7 @@ module Omnizip
             break if candidate.zero? || candidate >= ip
 
             dist = ip - candidate
-            break if dist >= BLOCK_MAX_SIZE
+            break if dist >= max_distance
             break if candidate + MIN_MATCH > size
 
             if src.byteslice(ip, MIN_MATCH) == src.byteslice(candidate, MIN_MATCH)
@@ -319,6 +346,15 @@ module Omnizip
             candidate = ms.chain[candidate]
           end
 
+          if ldm
+            lm = ldm.find_match(src, ip, max_distance, min_match,
+                                limit + min_match)
+            if lm && lm[1] > best_len
+              best_len = lm[1]
+              best_dist = lm[0]
+            end
+          end
+
           return nil if best_len < min_match
 
           [best_dist, best_len]
@@ -333,23 +369,39 @@ module Omnizip
         #
         # rubocop:disable Metrics/AbcSize
         # rubocop:disable Metrics/CyclomaticComplexity
-        def probe_match(src, ip, ms, min_match, limit)
+        def probe_match(src, ip, ms, min_match, limit, ldm = nil,
+                        max_distance = BLOCK_MAX_SIZE)
           size = src.bytesize
           return nil if ip + MIN_MATCH > size
 
           candidate = ms.hash_table[hash4(src, ip, ms.hash_log)]
-          return nil if candidate.zero? || candidate >= ip
 
-          dist = ip - candidate
-          return nil if dist >= BLOCK_MAX_SIZE
-          return nil if candidate + MIN_MATCH > size
-          return nil if src.byteslice(ip, MIN_MATCH) !=
-            src.byteslice(candidate, MIN_MATCH)
+          best_len = 0
+          best_dist = 0
+          if candidate.positive? && candidate < ip
+            dist = ip - candidate
+            if dist < max_distance && candidate + MIN_MATCH <= size &&
+                src.byteslice(ip, MIN_MATCH) == src.byteslice(candidate, MIN_MATCH)
+              best_len = MIN_MATCH + count_match(
+                src, ip + MIN_MATCH, src, candidate + MIN_MATCH,
+                [limit + MIN_MATCH_ECONOMICAL - ip - MIN_MATCH, 0].max
+              )
+              best_dist = dist
+            end
+          end
 
-          m_len = MIN_MATCH + count_match(src, ip + MIN_MATCH,
-                                          src, candidate + MIN_MATCH,
-                                          [limit + MIN_MATCH_ECONOMICAL - ip - MIN_MATCH, 0].max)
-          m_len >= min_match ? [dist, m_len] : nil
+          if ldm
+            lm = ldm.find_match(src, ip, max_distance, min_match,
+                                limit + min_match)
+            if lm && lm[1] > best_len
+              best_len = lm[1]
+              best_dist = lm[0]
+            end
+          end
+
+          return nil if best_len < min_match
+
+          [best_dist, best_len]
         end
         # rubocop:enable Metrics/AbcSize
         # rubocop:enable Metrics/CyclomaticComplexity

data/lib/omnizip/algorithms/zstandard.rb

@@ -57,6 +57,7 @@ module Omnizip
       autoload :SequencesEncoder,
                "omnizip/algorithms/zstandard/sequences_encoder"
       autoload :MatchFinder, "omnizip/algorithms/zstandard/match_finder"
+      autoload :LdmHashTable, "omnizip/algorithms/zstandard/ldm"
       autoload :XXHash64, "omnizip/algorithms/zstandard/xxhash"
 
       # Frame and FSE modules

data/lib/omnizip/implementations/seven_zip/lzma2/encoder.rb

@@ -20,506 +20,35 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-require "stringio"
-
 module Omnizip
   module Implementations
     module SevenZip
       module LZMA2
-        # 7-Zip SDK LZMA2 encoder implementation.
-        #
-        # This encoder produces LZMA2 compressed data compatible with 7-Zip format.
-        # It uses the same LZMA encoding logic as XZ Utils, but with 7-Zip
-        # format requirements (no EOS marker, no padding).
+        # 7-Zip LZMA2 encoder.
         #
-        # Key differences from XZ Utils implementation:
-        # - No EOS marker (raw LZMA2 data ends with 0x00 control byte)
-        # - No chunk padding (XZ pads to 4-byte boundary)
-        # - No LZMA2 property byte in data stream (method ID only in container)
+        # The LZMA2 stream itself is format-identical to the raw LZMA2
+        # layer of the XZ Utils encoder (chunks + 0x00 end marker); the
+        # 7-Zip container carries the method properties, so no property
+        # byte is written into the data stream (standalone: false).
         #
-        # Based on LZMA SDK by Igor Pavlov
-        # Reference: https://www.7-zip.org/sdk.html
-        #
-        # LZMA2 format (as used by 7-Zip):
-        # - Control byte specifies chunk type and dictionary reset
-        # - Dictionary size follows in some chunk types
-        # - Uncompressed size follows in some chunk types
-        # - Compressed data follows
-        class Encoder < Base::LZMA2EncoderBase
-          include Omnizip::Algorithms::LZMA2Const
-
-          # Maximum chunk sizes (from LZMA2 specification)
-          MAX_UNCOMPRESSED_CHUNK = 2 * 1024 * 1024 # 2MB
-          MAX_COMPRESSED_CHUNK = 64 * 1024 # 64KB
-
-          # Encoding constants
-          UINT32_MAX = 0xFFFFFFFF
-          REPS = 4
-          MATCH_LEN_MIN = 2
-
-          attr_reader :dict_size, :lc, :lp, :pb, :standalone
-
-          # Initialize 7-Zip SDK LZMA2 encoder
-          #
-          # @param dict_size [Integer] Dictionary size (must be power of 2)
-          # @param lc [Integer] Literal context bits (0-8)
-          # @param lp [Integer] Literal position bits (0-4)
-          # @param pb [Integer] Position bits (0-4)
-          # @param standalone [Boolean] Include property byte (false for 7-Zip)
-          def initialize(dict_size:, lc: 3, lp: 0, pb: 2, standalone: false)
-            super
-
-            # Initialize shared state across all chunks
-            # Using XZ Utils components (tested and working)
-
-            @dictionary = Omnizip::Algorithms::LZMA::Dictionary.new(dict_size)
-            @state = Omnizip::Algorithms::LZMA::LZMAState.new(0)
-            @models = Omnizip::Algorithms::LZMA::XzProbabilityModels.new(lc,
-                                                                         lp, pb)
-            @match_finder = Omnizip::Algorithms::LZMA::MatchFinder.new(@dictionary)
-            @optimal = Omnizip::Algorithms::LZMA::OptimalEncoder.new(mode: :fast)
-
-            # Track previous byte for literal context
-            @prev_byte = 0
-
-            # First chunk always resets dictionary (7-Zip compatibility)
-            @need_dictionary_reset = true
-            @need_state_reset = false
-            @need_properties = true
-          end
-
-          # Encode data with LZMA2 compression
-          #
-          # @param data [String] Input data to compress
-          # @return [String] LZMA2 compressed data (7-Zip format)
-          def encode(data)
-            return "" if data.empty?
-
-            output = StringIO.new
-            output.set_encoding(Encoding::BINARY)
-
-            # Write property byte if standalone mode
-            if @standalone
-              prop_byte = encode_dict_size(@dict_size)
-              output.putc(prop_byte)
-            end
-
-            # Reset match finder state for each encoding session
-            @match_finder.reset
-
-            # Process in chunks
-            input = StringIO.new(data)
-            input.set_encoding(Encoding::BINARY)
-
-            while !input.eof?
-              chunk_data = input.read(MAX_UNCOMPRESSED_CHUNK)
-              break if chunk_data.nil? || chunk_data.empty?
-
-              chunk = encode_chunk(chunk_data)
-              output.write(chunk)
-
-              # Update reset flags for next chunk
-              @need_dictionary_reset = false
-              @need_state_reset = false
-              @need_properties = false
-            end
-
-            # End of stream marker (0x00)
-            output.write(Omnizip::Algorithms::LZMA2::LZMA2Chunk.end_chunk.to_bytes)
-
-            output.string
-          end
-
-          # Get implementation identifier
+        # This is a thin subclass rather than a copy: the XZ Utils
+        # encoder carries the fixes for global pos_state across
+        # chunks, announced state resets, the 64 KiB compressed-chunk
+        # cap (16-bit size field) and the range-encoder symbol-queue
+        # drain, which a duplicated implementation silently missed.
+        class Encoder < Implementations::XZUtils::LZMA2::Encoder
+          # 7-Zip callers pass the same option keys; only the property
+          # byte default differs (the container describes the coder).
+          def initialize(options = {})
+            super({ standalone: false }.merge(options))
+          end
+
+          # Get implementation identifier.
           #
           # @return [Symbol] :seven_zip_sdk
           def implementation_name
             :seven_zip_sdk
           end
-
-          private
-
-          # Encode a single chunk with LZMA2 compression
-          #
-          # Uses XZ Utils encoding logic (tested and compatible)
-          def encode_chunk(uncompressed_data)
-            compressed = try_compress(uncompressed_data)
-
-            # Decide: compressed vs uncompressed
-            # Use compressed if it's actually smaller
-            if compressed.bytesize >= uncompressed_data.bytesize
-              # Use uncompressed chunk
-              chunk = Omnizip::Algorithms::LZMA2::LZMA2Chunk.new(
-                chunk_type: :uncompressed,
-                uncompressed_data: uncompressed_data,
-                compressed_data: "",
-                need_dict_reset: @need_dictionary_reset,
-                need_state_reset: false,
-                need_props: false,
-              )
-              # After uncompressed chunk, next chunk needs state reset
-              @need_state_reset = true
-            else
-              # Use compressed chunk
-              chunk_properties = (((@pb * 5) + @lp) * 9) + @lc
-              chunk = Omnizip::Algorithms::LZMA2::LZMA2Chunk.new(
-                chunk_type: :compressed,
-                uncompressed_data: uncompressed_data,
-                compressed_data: compressed,
-                compressed_size: compressed.bytesize,
-                properties: chunk_properties,
-                need_dict_reset: @need_dictionary_reset,
-                need_state_reset: @need_state_reset,
-                need_props: true,
-              )
-            end
-
-            # Update dictionary with the chunk data
-            @dictionary.append(uncompressed_data)
-
-            # Update prev_byte for next chunk
-            if uncompressed_data.bytesize.positive?
-              @prev_byte = uncompressed_data.getbyte(uncompressed_data.bytesize - 1)
-            end
-
-            chunk.to_bytes
-          end
-
-          # Try to compress data using LZMA
-          #
-          # Uses XZ Utils encoding components (tested and working)
-          def try_compress(data)
-            # Create output buffer
-            output_buffer = StringIO.new
-            output_buffer.set_encoding(Encoding::BINARY)
-
-            # Create range encoder
-            encoder = Omnizip::Algorithms::LZMA::XzRangeEncoder.new(output_buffer)
-
-            # Feed all data to match finder first
-            @match_finder.feed(data)
-
-            # Initialize hash table
-            match_len_max = 2
-            end_pos = [
-              @dictionary.buffer.bytesize + data.bytesize - match_len_max, 0
-            ].max
-            @match_finder.skip(end_pos)
-
-            # Position in match finder's buffer for encoding
-            start_pos = @dictionary.buffer.bytesize
-            @current_start_pos = start_pos
-
-            pos = 0
-            while pos < data.bytesize
-              # Encode queued symbols if buffer getting full
-              if encoder.count > 20
-                encode_queued_symbols(encoder, output_buffer)
-              end
-
-              # Find matches at current position
-              match_pos = start_pos + pos
-              @match_finder.find_matches(match_pos)
-
-              # Get optimal encoding choice
-              distance, length = @optimal.find_optimal(
-                match_pos,
-                @match_finder,
-                @state,
-                @state.reps,
-                @models,
-              )
-
-              # Encode based on choice
-              if distance == UINT32_MAX || length == 1
-                encode_literal(data.getbyte(pos), encoder, pos)
-                pos += 1
-              elsif distance < REPS
-                encode_repeated_match(distance, length, encoder, pos, match_pos)
-                pos += length
-              else
-                actual_distance = distance - REPS
-                encode_match(actual_distance, length, encoder, pos, match_pos,
-                             data)
-                pos += length
-              end
-            end
-
-            # Flush encoder
-            encode_queued_symbols(encoder, output_buffer)
-            encoder.queue_flush
-            encode_queued_symbols(encoder, output_buffer)
-
-            output_buffer.string
-          end
-
-          # Encode queued symbols to output
-          def encode_queued_symbols(encoder, output)
-            return if encoder.none?
-
-            temp_buffer = "\0" * 10000
-            out_pos = Omnizip::Algorithms::LZMA::IntRef.new(0)
-
-            size_before = output.size
-
-            encoder.encode_symbols(temp_buffer, out_pos, 10000)
-
-            if out_pos.value.positive?
-              output.write(StringCompat.byteslice(temp_buffer, 0,
-                                                  out_pos.value))
-            end
-
-            output.size - size_before
-          end
-
-          # Compatibility helper for Ruby 3.0-3.1
-          module StringCompat
-            if "".respond_to?(:byteslice)
-              def self.byteslice(string, start, length)
-                string.byteslice(start, length)
-              end
-            else
-              def self.byteslice(string, start, length)
-                string.bytes[start, length]&.pack("C*") || ""
-              end
-            end
-          end
-
-          # Encode literal byte
-          def encode_literal(symbol, encoder, pos)
-            pos_state = pos & ((1 << @pb) - 1)
-
-            prob_is_match = @models.is_match[@state.value][pos_state]
-            encoder.queue_bit(prob_is_match, 0)
-
-            literal_offset = get_literal_state(pos, @prev_byte)
-            use_matched = @state.use_matched_literal?
-
-            @state.update_literal!
-
-            if use_matched
-              match_pos = @current_start_pos + pos
-              match_byte_pos = match_pos - @state.reps[0] - 1
-              match_byte = @match_finder.buffer.getbyte(match_byte_pos) if match_byte_pos >= 0 && match_byte_pos < @match_finder.buffer.bytesize
-
-              if match_byte.nil?
-                encode_normal_literal(literal_offset, symbol, encoder)
-              else
-                encode_matched_literal(literal_offset, match_byte, symbol,
-                                       encoder)
-              end
-            else
-              encode_normal_literal(literal_offset, symbol, encoder)
-            end
-
-            @prev_byte = symbol
-          end
-
-          # Encode normal match
-          def encode_match(distance, length, encoder, pos, match_pos,
-_input_data)
-            pos_state = pos & ((1 << @pb) - 1)
-
-            prob_is_match = @models.is_match[@state.value][pos_state]
-            encoder.queue_bit(prob_is_match, 1)
-
-            prob_is_rep = @models.is_rep[@state.value]
-            encoder.queue_bit(prob_is_rep, 0)
-
-            # Reps are stored 0-based (distance - 1), matching the decoder's
-            # rep0 convention.
-            @state.update_match!(distance - 1)
-
-            encode_match_length(length, pos_state, encoder)
-            encode_distance(distance, length, encoder)
-
-            last_byte_pos = match_pos - distance + length - 1
-            @prev_byte = @match_finder.buffer.getbyte(last_byte_pos) if last_byte_pos >= 0 && last_byte_pos < @match_finder.buffer.bytesize
-          end
-
-          # Encode repeated match
-          def encode_repeated_match(rep, length, encoder, pos, match_pos)
-            pos_state = pos & ((1 << @pb) - 1)
-
-            prob_is_match = @models.is_match[@state.value][pos_state]
-            encoder.queue_bit(prob_is_match, 1)
-
-            prob_is_rep = @models.is_rep[@state.value]
-            encoder.queue_bit(prob_is_rep, 1)
-
-            prob_is_rep0 = @models.is_rep0[@state.value]
-            if rep.zero?
-              encoder.queue_bit(prob_is_rep0, 0)
-
-              prob_is_rep0_long = @models.is_rep0_long[@state.value][pos_state]
-              encoder.queue_bit(prob_is_rep0_long, length == 1 ? 0 : 1)
-            else
-              encoder.queue_bit(prob_is_rep0, 1)
-
-              prob_is_rep1 = @models.is_rep1[@state.value]
-              if rep == 1
-                encoder.queue_bit(prob_is_rep1, 0)
-              else
-                encoder.queue_bit(prob_is_rep1, 1)
-
-                prob_is_rep2 = @models.is_rep2[@state.value]
-                encoder.queue_bit(prob_is_rep2, rep - 2)
-              end
-
-              # XZ Utils rep_match: capture the selected distance BEFORE
-              # rotating, then shift reps down.
-              distance = @state.reps[rep]
-              rep.downto(1) { |i| @state.reps[i] = @state.reps[i - 1] }
-              @state.reps[0] = distance
-            end
-
-            if length == 1
-              @state.update_short_rep!
-            else
-              # Rep matches use their own length models (decoder's
-              # rep_length_coder).
-              encode_match_length(length, pos_state, encoder,
-                                  @models.rep_len_encoder)
-              @state.update_long_rep!
-            end
-
-            # Last byte of the match region in the stream
-            last_byte_pos = match_pos + length - 1
-            @prev_byte = @match_finder.buffer.getbyte(last_byte_pos) if last_byte_pos >= 0 && last_byte_pos < @match_finder.buffer.bytesize
-          end
-
-          def get_literal_state(pos, prev_byte)
-            literal_mask = (0x100 << @lp) - (0x100 >> @lc)
-            # Factor of 3: each literal context owns a 0x300-sized subcoder
-            # (XZ Utils literal_subcoder macro); must match the decoder's
-            # base_offset = 3 * (lit_state << lc).
-            3 * ((((pos << 8) + prev_byte) & literal_mask) << @lc)
-          end
-
-          def encode_normal_literal(literal_offset, symbol, encoder)
-            context = 1
-            8.downto(1) do |i|
-              bit = (symbol >> (i - 1)) & 1
-              encoder.queue_bit(@models.literal[literal_offset + context], bit)
-              context = (context << 1) | bit
-            end
-          end
-
-          def encode_matched_literal(literal_offset, match_byte, symbol,
-encoder)
-            offset = 0x100
-            symbol += 0x100
-
-            while symbol < 0x10000
-              match_byte <<= 1
-              match_bit = match_byte & offset
-              subcoder_index = offset + match_bit + (symbol >> 8)
-              bit = (symbol >> 7) & 1
-
-              encoder.queue_bit(
-                @models.literal[literal_offset + subcoder_index], bit
-              )
-
-              symbol <<= 1
-              offset &= ~(match_byte ^ symbol)
-            end
-          end
-
-          # len_encoder selects the length model set: match_len_encoder for
-          # normal matches, rep_len_encoder for repeated matches.
-          def encode_match_length(length, pos_state, encoder,
-                                  len_encoder = @models.match_len_encoder)
-            len = length - 2
-
-            if len < 8
-              encoder.queue_bit(len_encoder.choice, 0)
-              encode_bittree(len_encoder.low[pos_state], 3, len, encoder)
-            elsif len < 16
-              encoder.queue_bit(len_encoder.choice, 1)
-              encoder.queue_bit(len_encoder.choice2, 0)
-              encode_bittree(len_encoder.mid[pos_state], 3, len - 8, encoder)
-            else
-              encoder.queue_bit(len_encoder.choice, 1)
-              encoder.queue_bit(len_encoder.choice2, 1)
-              encode_bittree(len_encoder.high, 8, len - 16, encoder)
-            end
-          end
-
-          def encode_distance(distance, length, encoder)
-            # get_dist_slot expects a 0-based distance (xz fastpos.h:
-            # dist 4 -> slot 4, dist 5 -> slot 4).
-            dist0 = distance - 1
-            dist_slot = get_dist_slot(dist0)
-            len_state = [length - 2, 3].min
-
-            encode_bittree(@models.dist_slot[len_state], 6, dist_slot, encoder)
-
-            if dist_slot >= 4
-              footer_bits = (dist_slot >> 1) - 1
-              base = (2 | (dist_slot & 1)) << footer_bits
-              dist_reduced = dist0 - base
-
-              if dist_slot < 14
-                encode_bittree_reverse(@models.dist_special, dist_reduced,
-                                       footer_bits, base - dist_slot - 1, encoder)
-              else
-                direct_bits = footer_bits - 4
-                encoder.queue_direct_bits(dist_reduced >> 4, direct_bits)
-                align_mask = (1 << 4) - 1
-                encode_bittree_reverse(@models.dist_align,
-                                       dist_reduced & align_mask, 4, 0, encoder)
-              end
-            end
-          end
-
-          def encode_bittree(probs, num_bits, value, encoder)
-            context = 1
-            num_bits.downto(1) do |i|
-              bit = (value >> (i - 1)) & 1
-              encoder.queue_bit(probs[context], bit)
-              context = (context << 1) | bit
-            end
-          end
-
-          def encode_bittree_reverse(probs, value, num_bits, offset, encoder)
-            context = 1
-            num_bits.times do |i|
-              bit = (value >> i) & 1
-              encoder.queue_bit(probs[offset + context], bit)
-              context = (context << 1) | bit
-            end
-          end
-
-          def get_dist_slot(distance)
-            if distance < 4
-              distance
-            else
-              slot = 0
-              dist = distance
-              while dist > 3
-                dist >>= 1
-                slot += 2
-              end
-              slot + dist
-            end
-          end
-
-          def encode_dict_size(dict_size)
-            d = [dict_size, DICT_SIZE_MIN].max
-
-            log2_size = 0
-            temp = d
-            while temp > 1
-              log2_size += 1
-              temp >>= 1
-            end
-
-            if d == (1 << log2_size)
-              [(log2_size - 12) * 2, 40].min
-            else
-              [((log2_size - 12) * 2) + 1, 40].min
-            end
-          end
         end
       end
     end

data/lib/omnizip/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Omnizip
-  VERSION = "0.3.17"
+  VERSION = "0.3.19"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omnizip
 version: !ruby/object:Gem::Version
-  version: 0.3.17
+  version: 0.3.19
 platform: ruby
 authors:
 - Ribose Inc.
@@ -315,6 +315,7 @@ files:
 - lib/omnizip/algorithms/zstandard/fse/table_description.rb
 - lib/omnizip/algorithms/zstandard/huffman.rb
 - lib/omnizip/algorithms/zstandard/huffman_encoder.rb
+- lib/omnizip/algorithms/zstandard/ldm.rb
 - lib/omnizip/algorithms/zstandard/literals.rb
 - lib/omnizip/algorithms/zstandard/literals_encoder.rb
 - lib/omnizip/algorithms/zstandard/match_finder.rb