omnizip 0.3.17 → 0.3.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26bb587688afe9104a60ed3db30bd0b4d4a0ad909d1f02d0afaf3c8b67343119
-  data.tar.gz: 674f6bdf6fa36ec488ad06aa3f0a98ba792945fff3d244e9913ee06ff64402a6
+  metadata.gz: 689a5e44cef4327034541bbcdca28c15e75da2d6cb2452a0617365e107bf1597
+  data.tar.gz: d9bd5f70930e7efdb4fcaaba779927c0b0898e163e79e8e059a990ccc4083c31
 SHA512:
-  metadata.gz: ed59928f26cec56627b12304d9cf55d3ed04a7ee07f0a3bf2ef6f71b44a0e21d76d4246746db8f5dd0751139f68208a5a5a53c81ce9959f4433f883d3648adfc
-  data.tar.gz: 311d21ef3715257f1db58f9c1238200d04dce604022337fa53adf7a2c95d1a2d1272ba75bbcae55aaaf5d5b97a16ef954e7ab1b423e9f6875abb1f1036653a6b
+  metadata.gz: c2ec5fa49e00586c54e7a267de4da9fdcccacd0545e664e81608a5a697ad5430e95143ffd08afa0bafc41b44e17073d0780d694330748e7e35949991a77c9643
+  data.tar.gz: f79484e4aaa748152c1ea5abca31a08adcfe22207037608e3714e6fac57222c2d65763c16a0b26ef434a9f22ba8122ce4f27cddf9e5d4dc8e0db0c1b9abfe020

data/lib/omnizip/algorithms/zstandard/constants.rb

@@ -85,6 +85,10 @@ module Omnizip
         MAX_LEVEL = 22
         DEFAULT_LEVEL = 3
 
+        # Long-distance matching kicks in at this level for inputs
+        # larger than one block (Btultra2 in the C strategy table).
+        LDM_MIN_LEVEL = 19
+
         # Buffer size for streaming operations
         BUFFER_SIZE = 128 * 1024 # 128KB
 

data/lib/omnizip/algorithms/zstandard/encoder.rb

@@ -95,6 +95,21 @@ module Omnizip
           ms = MatchFinder::MatchState.new(params[:hash_log])
           ms.enable_chain(params[:chain]) if params[:chain].positive?
 
+          # LDM (long-distance matching) at high levels on multi-block
+          # inputs: a sparse table over the whole frame finds matches
+          # beyond the 128 KiB block window. Chains are disabled —
+          # the chain table is block-sized and LDM provides the
+          # long-distance coverage instead (Rust reference behavior).
+          ldm = nil
+          max_distance = BLOCK_MAX_SIZE
+          if @level >= LDM_MIN_LEVEL && data.bytesize > BLOCK_MAX_SIZE
+            window_log = data.bytesize.bit_length.clamp(10, 31)
+            ldm = LdmHashTable.new(data.bytesize, window_log)
+            (0...data.bytesize).each { |pos| ldm.insert(data, pos) }
+            ms.disable_chain
+            max_distance = data.bytesize
+          end
+
           # Wire repeat-offset state carried across blocks; matches the
           # decoder's executor state. Raw and RLE blocks leave it
           # untouched.
@@ -114,7 +129,7 @@ module Omnizip
               seq_store = MatchFinder::SeqStore.new(reps.dup)
               MatchFinder.compress_range(data, offset, block_end, seq_store,
                                          ms, params[:min_match],
-                                         params[:lazy])
+                                         params[:lazy], ldm, max_distance)
               content, new_reps = try_compressed(seq_store, reps)
               if content.nil?
                 write_block_header(is_last ? 1 : 0, BLOCK_TYPE_RAW,

data/lib/omnizip/algorithms/zstandard/ldm.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Long-Distance Matching sparse hash table (port of the
+      # omnizip-rs encoder/ldm.rs).
+      #
+      # Hashes every `gap`-th position into a head/chain table so the
+      # parser can find matches at distances far beyond the normal
+      # match-finder window — up to the full frame size. The table is
+      # pre-populated over the whole input before parsing starts, so
+      # entries may point forward of the current position; find_match
+      # skips those without spending its chain budget.
+      class LdmHashTable
+        # Chain entries walked per find_match (Rust LDM_MAX_CHAIN).
+        LDM_MAX_CHAIN = 32
+        MIN_MATCH = MatchFinder::MIN_MATCH
+
+        # Hash table and chain sizes are additionally capped by the
+        # number of sparse samples so small inputs do not pay for a
+        # window-sized table.
+        def initialize(input_len, window_log, gap = 16)
+          samples = (input_len / gap) + 1
+          hash_log = [[window_log, 21].min, samples.bit_length + 2].min
+          hash_log = 1 if hash_log < 1
+
+          @gap = gap
+          @hash_log = hash_log
+          @head = Array.new(1 << hash_log)
+          @chain = Array.new(samples)
+        end
+
+        # Insert `pos` into the table (sparse sampling: every gap-th
+        # position only).
+        def insert(src, pos)
+          return unless (pos % @gap).zero?
+
+          h = hash4(src, pos)
+          idx = pos / @gap
+          @chain[idx] = @head[h] if idx < @chain.length
+          @head[h] = pos
+        end
+
+        # Longest match at `pos` within `max_distance`, bounded by
+        # `end_bound` (the current block end). Walks up to MAX_CHAIN
+        # chain entries. Returns [distance, length] or nil.
+        #
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/AbcSize
+        # rubocop:disable-next Metrics/CyclomaticComplexity
+        def find_match(src, pos, max_distance, min_match, end_bound)
+          return nil if pos + MIN_MATCH > src.bytesize
+
+          candidate = @head[hash4(src, pos)]
+          best_len = 0
+          best_dist = 0
+          chain_count = 0
+
+          while candidate && chain_count < LDM_MAX_CHAIN
+            if candidate >= pos
+              candidate = @chain[candidate / @gap]
+              next
+            end
+
+            dist = pos - candidate
+            break if dist > max_distance
+
+            len_cap = [end_bound - pos, pos - candidate].min
+            len_cap = 0 if len_cap.negative?
+            len = MatchFinder.count_match(src, pos, src, candidate, len_cap)
+            if len > best_len && len >= min_match
+              best_len = len
+              best_dist = dist
+            end
+
+            candidate = @chain[candidate / @gap]
+            chain_count += 1
+          end
+
+          return nil unless best_len >= min_match && best_dist.positive?
+
+          [best_dist, best_len]
+        end
+        # rubocop:enable Metrics/AbcSize
+        # rubocop:enable Metrics/MethodLength
+
+        private
+
+        # rubocop:disable-next Metrics/AbcSize
+        def hash4(src, pos)
+          v = src.getbyte(pos) |
+            (src.getbyte(pos + 1) << 8) |
+            (src.getbyte(pos + 2) << 16) |
+            (src.getbyte(pos + 3) << 24)
+          ((v * MatchFinder::PRIME4_BYTES) & 0xFFFFFFFF) >> (32 - @hash_log)
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/match_finder.rb

@@ -163,7 +163,8 @@ module Omnizip
         # rubocop:disable Metrics/CyclomaticComplexity
         # rubocop:disable-next Metrics/PerceivedComplexity
         def compress_range(src, block_start, block_end, seq_store, ms,
-                           min_match, lazy)
+                           min_match, lazy, ldm = nil,
+                           max_distance = BLOCK_MAX_SIZE)
           mm = [min_match, MIN_MATCH_ECONOMICAL].max
           span = block_end - block_start
           if span < mm + 1
@@ -177,7 +178,11 @@ module Omnizip
           limit = block_end - mm
 
           while ip < limit
-            match = if lazy.zero?
+            match = if ldm
+                      find_match_ldm(src, ip, ms, mm, limit, anchor,
+                                     seq_store.rep_offsets[0], ldm,
+                                     max_distance)
+                    elsif lazy.zero?
                       find_greedy_match(src, ip, ms, mm, limit, anchor,
                                         seq_store.rep_offsets[0])
                     else
@@ -190,7 +195,12 @@ module Omnizip
                 (1..lazy).each do |k|
                   next unless ip + k < limit
 
-                  m2 = probe_match(src, ip + k, ms, mm, limit)
+                  m2 = if ldm
+                         probe_match(src, ip + k, ms, mm, limit, ldm,
+                                     max_distance)
+                       else
+                         probe_match(src, ip + k, ms, mm, limit)
+                       end
                   if m2 && m2[1] > len + k
                     defer = true
                     break
@@ -224,6 +234,22 @@ module Omnizip
         # rubocop:enable Metrics/AbcSize
         # rubocop:enable Metrics/MethodLength
 
+        # LDM-mode match search (port of the Rust
+        # compress_block_lazy2_with_ldm loop body): rep0 fast-path
+        # first, then the normal hash probe merged with the LDM
+        # table. Returns [offset, length, start] or nil.
+        def find_match_ldm(src, ip, ms, min_match, limit, anchor, rep0,
+                           ldm, max_distance)
+          m = rep0_match(src, ip, rep0, min_match, limit, anchor)
+          return m if m
+
+          dist, len = find_best_match(src, ip, ms, min_match, limit,
+                                      ldm, max_distance)
+          return nil unless dist
+
+          [dist, len, ip]
+        end
+
         # Greedy-path match search (port of the Rust
         # compress_block_with_min_match): the rep0 fast-path first
         # (cheapest offset to encode), then the hash table — both
@@ -280,7 +306,8 @@ module Omnizip
         # rubocop:disable Metrics/MethodLength
         # rubocop:disable Metrics/AbcSize
         # rubocop:disable Metrics/CyclomaticComplexity
-        def find_best_match(src, ip, ms, min_match, limit)
+        def find_best_match(src, ip, ms, min_match, limit, ldm = nil,
+                            max_distance = BLOCK_MAX_SIZE)
           size = src.bytesize
           return nil if ip + MIN_MATCH > size
 
@@ -300,7 +327,7 @@ module Omnizip
             break if candidate.zero? || candidate >= ip
 
             dist = ip - candidate
-            break if dist >= BLOCK_MAX_SIZE
+            break if dist >= max_distance
             break if candidate + MIN_MATCH > size
 
             if src.byteslice(ip, MIN_MATCH) == src.byteslice(candidate, MIN_MATCH)
@@ -319,6 +346,15 @@ module Omnizip
             candidate = ms.chain[candidate]
           end
 
+          if ldm
+            lm = ldm.find_match(src, ip, max_distance, min_match,
+                                limit + min_match)
+            if lm && lm[1] > best_len
+              best_len = lm[1]
+              best_dist = lm[0]
+            end
+          end
+
           return nil if best_len < min_match
 
           [best_dist, best_len]
@@ -333,23 +369,39 @@ module Omnizip
         #
         # rubocop:disable Metrics/AbcSize
         # rubocop:disable Metrics/CyclomaticComplexity
-        def probe_match(src, ip, ms, min_match, limit)
+        def probe_match(src, ip, ms, min_match, limit, ldm = nil,
+                        max_distance = BLOCK_MAX_SIZE)
           size = src.bytesize
           return nil if ip + MIN_MATCH > size
 
           candidate = ms.hash_table[hash4(src, ip, ms.hash_log)]
-          return nil if candidate.zero? || candidate >= ip
 
-          dist = ip - candidate
-          return nil if dist >= BLOCK_MAX_SIZE
-          return nil if candidate + MIN_MATCH > size
-          return nil if src.byteslice(ip, MIN_MATCH) !=
-            src.byteslice(candidate, MIN_MATCH)
+          best_len = 0
+          best_dist = 0
+          if candidate.positive? && candidate < ip
+            dist = ip - candidate
+            if dist < max_distance && candidate + MIN_MATCH <= size &&
+                src.byteslice(ip, MIN_MATCH) == src.byteslice(candidate, MIN_MATCH)
+              best_len = MIN_MATCH + count_match(
+                src, ip + MIN_MATCH, src, candidate + MIN_MATCH,
+                [limit + MIN_MATCH_ECONOMICAL - ip - MIN_MATCH, 0].max
+              )
+              best_dist = dist
+            end
+          end
 
-          m_len = MIN_MATCH + count_match(src, ip + MIN_MATCH,
-                                          src, candidate + MIN_MATCH,
-                                          [limit + MIN_MATCH_ECONOMICAL - ip - MIN_MATCH, 0].max)
-          m_len >= min_match ? [dist, m_len] : nil
+          if ldm
+            lm = ldm.find_match(src, ip, max_distance, min_match,
+                                limit + min_match)
+            if lm && lm[1] > best_len
+              best_len = lm[1]
+              best_dist = lm[0]
+            end
+          end
+
+          return nil if best_len < min_match
+
+          [best_dist, best_len]
         end
         # rubocop:enable Metrics/AbcSize
         # rubocop:enable Metrics/CyclomaticComplexity

data/lib/omnizip/algorithms/zstandard.rb

@@ -57,6 +57,7 @@ module Omnizip
       autoload :SequencesEncoder,
                "omnizip/algorithms/zstandard/sequences_encoder"
       autoload :MatchFinder, "omnizip/algorithms/zstandard/match_finder"
+      autoload :LdmHashTable, "omnizip/algorithms/zstandard/ldm"
       autoload :XXHash64, "omnizip/algorithms/zstandard/xxhash"
 
       # Frame and FSE modules

data/lib/omnizip/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Omnizip
-  VERSION = "0.3.17"
+  VERSION = "0.3.18"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omnizip
 version: !ruby/object:Gem::Version
-  version: 0.3.17
+  version: 0.3.18
 platform: ruby
 authors:
 - Ribose Inc.
@@ -315,6 +315,7 @@ files:
 - lib/omnizip/algorithms/zstandard/fse/table_description.rb
 - lib/omnizip/algorithms/zstandard/huffman.rb
 - lib/omnizip/algorithms/zstandard/huffman_encoder.rb
+- lib/omnizip/algorithms/zstandard/ldm.rb
 - lib/omnizip/algorithms/zstandard/literals.rb
 - lib/omnizip/algorithms/zstandard/literals_encoder.rb
 - lib/omnizip/algorithms/zstandard/match_finder.rb