omnizip 0.3.21 → 0.3.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d0052490f16345b27296100499706bddd9936ea4a4b839dc34ea862bef32d77
-  data.tar.gz: f33f511d967b2a4f5d881b0e9205c72e57a9744480fdcb914b5ee44df9a14983
+  metadata.gz: 61cf26f9e324600f286b236ab80e02d720059f32bca2fa3c71640ba3abc4355a
+  data.tar.gz: fa189e081e402ecb4d084490e7874402faacf231a296cd009a15f218fcd9453e
 SHA512:
-  metadata.gz: 722b0ece40f17d907a74df2fd9a333945f5060101f6c816bc613c56b1e653d907b31fc803afe40b840217e70dd6301304e0c39660071a0fab20d3df4e6a9c884
-  data.tar.gz: f3810c7f1d35b12a88dbc2651945ff18d66399b42f5bb3db1299aece2545fceb9e6336025d2e3555d4b8aa4608f3991091ddd33dca0d575a1b6153da0b4c6a02
+  metadata.gz: 308660d6095a909d72c528bdfb5e828e76d320172b979a53ef7401aa2c1c12a560a0afd57f49991eb5190f87deca1c15cd798e8abae89bd3341ed3bb43c39b3d
+  data.tar.gz: 29ac9066539f2d8cef3973a2eb9cfe15a549a5efead04a1f5c88f28587fb6a2d23ac28c401c1c4187f5c226bfe4b531e3565523400dcd183adc83c842cd2d5dc

data/CHANGELOG.md

@@ -7,6 +7,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.23] - 2026-08-26
+
+### Changed
+- Zstd encoder hot paths use allocation-free integer reads instead of
+  `String#byteslice` + `unpack1` comparisons (profiling showed GC at
+  52% of encoder time): default-level compression is ~3.4x faster and
+  level 22 ~1.9x on the benchmark corpus, with byte-identical output.
+
+## [0.3.22] - 2026-08-25
+
+### Added
+- Zstandard dictionary compression (`Dictionary.from_raw` /
+  `serialize` / `deserialize`, `compress_with_dict`,
+  `decompress_with_dict`): the dictionary content primes the match
+  finder as shared history and the frame header carries the
+  Dictionary_ID, which the decoder verifies (Phase-1 scope, as in the
+  Rust reference; entropy-table preloading is future work).
+
+## [0.3.21] - 2026-08-25
+
+### Fixed
+- 7-Zip encoder dictionary is now capped at the size announced in the
+  coder properties (prevents matches reading outside the decoder
+  window for inputs above the announced size).
+
+### Changed
+- Zstd lazy levels (6+) gained the rep0 fast-path and backward
+  extension: levels 6-12 improve 0.180 -> 0.154 and levels 19-22
+  0.146 -> 0.126 on the benchmark corpus; the level scale is now
+  monotonic.
+- Zstd adaptive block splitting: heterogeneous chunks of 32 KiB or
+  more split into 16 KiB sub-blocks so entropy tables fit each
+  content regime.
+
+## [0.3.20] - 2026-08-25
+
+### Removed
+- Dead `SevenZipLZMA2`/`XZLZMA2` algorithm wrappers (never
+  autoloaded; one referenced a nonexistent decoder constant).
+
 ## [0.3.19] - 2026-08-25
 
 ### Fixed

data/lib/omnizip/algorithms/zstandard/decoder.rb

@@ -79,6 +79,35 @@ module Omnizip
           output
         end
 
+        # Decode a stream produced by a dictionary-primed encoder: the
+        # dictionary's content primes the reconstruction window of
+        # frames carrying its ID (verified; mismatch raises).
+        #
+        # @param dict [Dictionary]
+        # @return [String] decompressed data (binary)
+        def decode_stream_with_dict(dict)
+          data = read_all
+          output = String.new(encoding: Encoding::BINARY)
+          pos = 0
+
+          loop do
+            remaining = data.bytesize - pos
+            break if remaining.zero?
+            raise Omnizip::DecompressionError, "trailing bytes are not a frame" if remaining < 4
+
+            magic = data.byteslice(pos, 4).unpack1("V")
+            unless magic == MAGIC_NUMBER
+              raise Omnizip::DecompressionError,
+                    "invalid Zstandard magic: 0x#{magic.to_s(16)}"
+            end
+
+            frame_output, pos = decode_frame(data, pos + 4, dict)
+            output << frame_output
+          end
+
+          output
+        end
+
         private
 
         def read_all
@@ -101,8 +130,15 @@ module Omnizip
 
         # Decode one frame (input positioned after the magic).
         # Returns [frame_output, pos_after_frame].
-        def decode_frame(data, pos)
+        #
+        # With a dictionary prefix, the reconstruction window is
+        # primed with the dictionary content so sequences can
+        # back-reference it; the prefix is verified (checksum) and
+        # stripped from the returned output.
+        def decode_frame(data, pos, dict = nil)
           header, pos = Frame::Header.parse_from(data, pos)
+          verify_dictionary_id(header, dict)
+          prefix = dict&.content
 
           # Reset per-frame state: repeat offsets, previous Huffman
           # table, previous FSE tables.
@@ -111,6 +147,7 @@ module Omnizip
           @previous_fse_tables = {}
 
           output = String.new(encoding: Encoding::BINARY)
+          output << prefix if prefix
           loop do
             block, pos = Frame::Block.parse_from(data, pos)
             if block.reserved?
@@ -128,7 +165,9 @@ module Omnizip
             end
 
             expected = data.byteslice(pos, 4).unpack1("V")
-            actual = XXHash64.frame_checksum(output)
+            actual = XXHash64.frame_checksum(
+              prefix ? output.byteslice(prefix.bytesize..) : output,
+            )
             if expected != actual
               raise Omnizip::DecompressionError,
                     "frame checksum mismatch: stored 0x#{expected.to_s(16)}, " \
@@ -137,9 +176,29 @@ module Omnizip
             pos += 4
           end
 
-          [output, pos]
+          frame_output = prefix ? output.byteslice(prefix.bytesize..) : output
+          [frame_output, pos]
+        end
+
+        # A frame that carries a Dictionary_ID must be decoded with a
+        # dictionary of the same ID; a mismatched or missing
+        # dictionary would silently garble every back-reference.
+        def verify_dictionary_id(header, dict)
+          return unless header.dictionary_id?
+
+          frame_id = header.dictionary_id
+          if dict.nil?
+            raise Omnizip::DecompressionError,
+                  "frame requires dictionary #{frame_id}, none supplied"
+          end
+          return if dict.id == frame_id
+
+          raise Omnizip::DecompressionError,
+                "frame requires dictionary #{frame_id}, got #{dict.id}"
         end
 
+        # Decode one block into `output`. Returns the position after
+        # the block's payload.
         def decode_block(block, data, pos, output)
           case block.block_type
           when BLOCK_TYPE_RAW

data/lib/omnizip/algorithms/zstandard/dictionary.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+module Omnizip
+  module Algorithms
+    class Zstandard
+      # Zstandard dictionary (port of the omnizip-rs dict.rs).
+      #
+      # A dictionary lets the encoder preload a reference-content
+      # window so small inputs compress dramatically better: the
+      # dictionary content is used as a match-finder prefix, and the
+      # frame header carries the dictionary's ID.
+      #
+      # Wire format (simplified form, as in the Rust reference):
+      # magic (4) + dictionary ID (4, LE) + raw content.
+      class Dictionary
+        DICT_MAGIC = 0xEC30A437
+
+        attr_reader :id, :content
+
+        # @param id [Integer] dictionary ID carried in frame headers
+        # @param content [String] corpus bytes used as the prefix
+        def initialize(id, content)
+          @id = id
+          @content = content.dup.force_encoding(Encoding::BINARY)
+        end
+
+        def self.from_raw(id, content)
+          new(id, content)
+        end
+
+        def self.deserialize(data)
+          if data.bytesize < 8
+            raise Omnizip::DecompressionError,
+                  "dictionary too short for magic + id"
+          end
+          if data.byteslice(0, 4).unpack1("V") != DICT_MAGIC
+            raise Omnizip::DecompressionError, "bad dictionary magic"
+          end
+
+          new(data.byteslice(4, 4).unpack1("V"), data.byteslice(8..))
+        end
+
+        def serialize
+          [DICT_MAGIC].pack("V") + [@id].pack("V") + @content
+        end
+
+        def ==(other)
+          other.is_a?(Dictionary) && id == other.id && content == other.content
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/algorithms/zstandard/encoder.rb

@@ -58,42 +58,83 @@ module Omnizip
           write_frame(data)
         end
 
+        # Encode a data stream primed with a dictionary prefix (port
+        # of the Rust encode_frame_with_dict): the dictionary content
+        # is prepended as virtual history, the match finder is seeded
+        # with its positions, and only the plaintext is emitted.
+        # Frame_Content_Size and the checksum cover the plaintext
+        # only; the frame header carries the dictionary's ID and
+        # requires the dict-aware decoder.
+        #
+        # @param data [String]
+        # @param dict [Dictionary]
+        def encode_stream_with_dict(data, dict)
+          data = data.dup.force_encoding(Encoding::BINARY)
+          virtual = dict.content + data
+          prefix_len = dict.content.bytesize
+
+          write_u32le(MAGIC_NUMBER)
+          write_frame_header(data, dict.id)
+          write_blocks(virtual, prefix_len)
+          write_u32le(XXHash64.frame_checksum(data)) if @checksum
+        end
+
         private
 
-        def write_frame(data)
+        def write_frame(data, dict = nil)
           write_u32le(MAGIC_NUMBER)
-          write_frame_header(data)
+          write_frame_header(data, dict && dict.id)
           write_blocks(data)
           write_u32le(XXHash64.frame_checksum(data)) if @checksum
         end
 
-        # Single-segment frame header: no window descriptor, no
-        # dictionary; Frame_Content_Size in 1, 2, 4 or 8 bytes.
-        def write_frame_header(data)
+        # Single-segment frame header: no window descriptor;
+        # optional Dictionary_ID, then Frame_Content_Size in 1, 2, 4
+        # or 8 bytes.
+        def write_frame_header(data, dict_id = nil)
+          did_flag, did_bytes = dict_id_encoding(dict_id)
+
           size = data.bytesize
           if size < 256
-            @output_stream.putc(0x20) # single segment, FCS 1 byte
+            @output_stream.putc(0x20 | did_flag) # single segment, FCS 1 byte
+            write_dict_id_bytes(did_bytes)
             @output_stream.putc(size)
           elsif size <= 65_791
-            @output_stream.putc(0x60) # single segment, FCS 2 bytes (+256)
+            @output_stream.putc(0x60 | did_flag) # single segment, FCS 2 bytes
+            write_dict_id_bytes(did_bytes)
             write_u16le(size - 256)
           elsif size <= 0xFFFFFFFF
-            @output_stream.putc(0xA0) # single segment, FCS 4 bytes
+            @output_stream.putc(0xA0 | did_flag) # single segment, FCS 4 bytes
+            write_dict_id_bytes(did_bytes)
             write_u32le(size)
           else
-            @output_stream.putc(0xE0) # single segment, FCS 8 bytes
+            @output_stream.putc(0xE0 | did_flag) # single segment, FCS 8 bytes
+            write_dict_id_bytes(did_bytes)
             write_u64le(size)
           end
         end
 
+        def dict_id_encoding(dict_id)
+          return [0, "".b] if dict_id.nil? || dict_id.zero?
+          return [1, [dict_id].pack("C")] if dict_id <= 0xFF
+          return [2, [dict_id].pack("v")] if dict_id <= 0xFFFF
+
+          [3, [dict_id].pack("V")]
+        end
+
+        def write_dict_id_bytes(bytes)
+          @output_stream.write(bytes)
+        end
+
         # rubocop:disable Metrics/MethodLength
         # rubocop:disable-next Metrics/AbcSize
-        def write_blocks(data)
-          return write_empty_last_block if data.empty?
+        def write_blocks(data, start = 0)
+          return write_empty_last_block if data.bytesize <= start
 
           params = MatchFinder.params_for_level(@level, data.bytesize)
           ms = MatchFinder::MatchState.new(params[:hash_log])
           ms.enable_chain(params[:chain]) if params[:chain].positive?
+          ms.seed_prefix(data, start) if start.positive?
 
           # LDM (long-distance matching) at high levels on multi-block
           # inputs: a sparse table over the whole frame finds matches
@@ -115,7 +156,7 @@ module Omnizip
           # untouched.
           reps = [1, 4, 8]
 
-          offset = 0
+          offset = start
           while offset < data.bytesize
             block_end = [offset + BLOCK_MAX_SIZE, data.bytesize].min
             is_last = block_end == data.bytesize

data/lib/omnizip/algorithms/zstandard/match_finder.rb

@@ -88,6 +88,18 @@ module Omnizip
           def disable_chain
             @max_chain = 0
           end
+
+          # Seed the hash table with a prefix's positions (dictionary
+          # content prepended to the plaintext): the most recent
+          # position per hash wins, matching insert order.
+          def seed_prefix(src, prefix_len)
+            return if prefix_len < MIN_MATCH
+
+            limit = prefix_len - MIN_MATCH + 1
+            (0...limit).each do |pos|
+              @hash_table[MatchFinder.hash4(src, pos, @hash_log)] = pos
+            end
+          end
         end
 
         module_function
@@ -119,11 +131,29 @@ module Omnizip
             min_match: MIN_MATCH_ECONOMICAL }
         end
 
-        def hash4(src, pos, h_bits)
-          v = src.getbyte(pos) |
+        # Allocation-free little-endian reads; the hot loops call
+        # these millions of times, and String#byteslice + unpack1
+        # showed up as half the encoder's time via GC pressure.
+        def read4(src, pos)
+          src.getbyte(pos) |
             (src.getbyte(pos + 1) << 8) |
             (src.getbyte(pos + 2) << 16) |
             (src.getbyte(pos + 3) << 24)
+        end
+
+        def read8(src, pos)
+          src.getbyte(pos) |
+            (src.getbyte(pos + 1) << 8) |
+            (src.getbyte(pos + 2) << 16) |
+            (src.getbyte(pos + 3) << 24) |
+            (src.getbyte(pos + 4) << 32) |
+            (src.getbyte(pos + 5) << 40) |
+            (src.getbyte(pos + 6) << 48) |
+            (src.getbyte(pos + 7) << 56)
+        end
+
+        def hash4(src, pos, h_bits)
+          v = read4(src, pos)
           # 32-bit wrapping multiply (C/uint32_t), then the high bits.
           ((v * PRIME4_BYTES) & 0xFFFFFFFF) >> (32 - h_bits)
         end
@@ -136,8 +166,8 @@ module Omnizip
           b_size = b.bytesize
           while len + 8 <= limit && a_pos + len + 8 <= a_size &&
               b_pos + len + 8 <= b_size
-            wa = a.byteslice(a_pos + len, 8).unpack1("Q<")
-            wb = b.byteslice(b_pos + len, 8).unpack1("Q<")
+            wa = read8(a, a_pos + len)
+            wb = read8(b, b_pos + len)
             if wa == wb
               len += 8
             else
@@ -277,8 +307,7 @@ module Omnizip
         # rubocop:disable-next Metrics/AbcSize
         def rep0_match(src, ip, rep0, min_match, limit, anchor)
           return nil if rep0 <= 0 || ip <= rep0
-          return nil if src.byteslice(ip, MIN_MATCH) !=
-            src.byteslice(ip - rep0, MIN_MATCH)
+          return nil if read4(src, ip) != read4(src, ip - rep0)
 
           m_len = MIN_MATCH + count_match(
             src, ip + MIN_MATCH, src, ip + MIN_MATCH - rep0,
@@ -334,7 +363,7 @@ module Omnizip
             break if dist >= max_distance
             break if candidate + MIN_MATCH > size
 
-            if src.byteslice(ip, MIN_MATCH) == src.byteslice(candidate, MIN_MATCH)
+            if read4(src, ip) == read4(src, candidate)
               m_len = MIN_MATCH + count_match(src, ip + MIN_MATCH,
                                               src, candidate + MIN_MATCH,
                                               [max_extend - MIN_MATCH, 0].max)
@@ -385,7 +414,7 @@ module Omnizip
           if candidate.positive? && candidate < ip
             dist = ip - candidate
             if dist < max_distance && candidate + MIN_MATCH <= size &&
-                src.byteslice(ip, MIN_MATCH) == src.byteslice(candidate, MIN_MATCH)
+                read4(src, ip) == read4(src, candidate)
               best_len = MIN_MATCH + count_match(
                 src, ip + MIN_MATCH, src, candidate + MIN_MATCH,
                 [limit + MIN_MATCH_ECONOMICAL - ip - MIN_MATCH, 0].max

data/lib/omnizip/algorithms/zstandard.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "stringio"
+
 # Copyright (C) 2025 Ribose Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
@@ -58,6 +60,7 @@ module Omnizip
                "omnizip/algorithms/zstandard/sequences_encoder"
       autoload :MatchFinder, "omnizip/algorithms/zstandard/match_finder"
       autoload :LdmHashTable, "omnizip/algorithms/zstandard/ldm"
+      autoload :Dictionary, "omnizip/algorithms/zstandard/dictionary"
       autoload :XXHash64, "omnizip/algorithms/zstandard/xxhash"
 
       # Frame and FSE modules
@@ -101,6 +104,54 @@ module Omnizip
         output_stream.write(decompressed)
       end
 
+      # Compress data primed with a dictionary prefix: the dictionary
+      # content acts as shared history the match finder can reference,
+      # dramatically improving ratios on small inputs. The resulting
+      # frame carries the dictionary's ID and requires
+      # decompress_with_dict to decode.
+      #
+      # @param input_stream [IO] plaintext
+      # @param output_stream [IO] compressed frame
+      # @param dict [Dictionary]
+      # @param options [Hash] :level, :checksum
+      # @return [void]
+      def compress_with_dict(input_stream, output_stream, dict, options = {})
+        encoder = Encoder.new(output_stream, build_encoder_options(options))
+        encoder.encode_stream_with_dict(input_stream.read, dict)
+      end
+
+      # Decompress a frame produced by compress_with_dict. The frame's
+      # dictionary ID is verified against `dict` before decoding.
+      #
+      # @param input_stream [IO] compressed frame
+      # @param output_stream [IO] plaintext
+      # @param dict [Dictionary]
+      # @return [void]
+      def decompress_with_dict(input_stream, output_stream, dict)
+        output_stream.set_encoding(Encoding::BINARY)
+        decoder = Decoder.new(input_stream)
+        output_stream.write(decoder.decode_stream_with_dict(dict))
+      end
+
+      # Class-level convenience mirrors of the dictionary API.
+      #
+      # @return [String] compressed frame / decompressed plaintext
+      def self.compress_with_dict(data, dict, **options)
+        output = StringIO.new
+        output.set_encoding(Encoding::BINARY)
+        new(options).compress_with_dict(
+          StringIO.new(data.to_s.b), output, dict, options
+        )
+        output.string
+      end
+
+      def self.decompress_with_dict(data, dict)
+        output = StringIO.new
+        output.set_encoding(Encoding::BINARY)
+        new.decompress_with_dict(StringIO.new(data.to_s.b), output, dict)
+        output.string
+      end
+
       private
 
       # Build encoder options from compression options

data/lib/omnizip/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Omnizip
-  VERSION = "0.3.21"
+  VERSION = "0.3.23"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omnizip
 version: !ruby/object:Gem::Version
-  version: 0.3.21
+  version: 0.3.23
 platform: ruby
 authors:
 - Ribose Inc.
@@ -301,6 +301,7 @@ files:
 - lib/omnizip/algorithms/zstandard.rb
 - lib/omnizip/algorithms/zstandard/constants.rb
 - lib/omnizip/algorithms/zstandard/decoder.rb
+- lib/omnizip/algorithms/zstandard/dictionary.rb
 - lib/omnizip/algorithms/zstandard/encoder.rb
 - lib/omnizip/algorithms/zstandard/frame.rb
 - lib/omnizip/algorithms/zstandard/frame/block.rb