omnizip 0.3.18 → 0.3.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 689a5e44cef4327034541bbcdca28c15e75da2d6cb2452a0617365e107bf1597
-  data.tar.gz: d9bd5f70930e7efdb4fcaaba779927c0b0898e163e79e8e059a990ccc4083c31
+  metadata.gz: affda0906d52a418254670adb02e96312d6e268d1df4aafbba575ade503edfbe
+  data.tar.gz: ae4e84af21221350bf4d67a1b295546674527e091b500a715cb96f8547870990
 SHA512:
-  metadata.gz: c2ec5fa49e00586c54e7a267de4da9fdcccacd0545e664e81608a5a697ad5430e95143ffd08afa0bafc41b44e17073d0780d694330748e7e35949991a77c9643
-  data.tar.gz: f79484e4aaa748152c1ea5abca31a08adcfe22207037608e3714e6fac57222c2d65763c16a0b26ef434a9f22ba8122ce4f27cddf9e5d4dc8e0db0c1b9abfe020
+  metadata.gz: 91044c5b1067662c5177e2b8ca262ad0434e4a4cf94e9ea321251723499188fc4b57b3793e2ed88ce51ea406cdb1bfb6d31921ef3b08a7baaf4fd6e62102dc5b
+  data.tar.gz: b6614848a2eb450b748bca1d3d297b7bcd5751607124687237c75ebdeeb43b033ffe6d9af49af457e94aa1cace5ab615c3fe6d04f6a5bb3206a863249225ead6

data/lib/omnizip/implementations/seven_zip/lzma2/encoder.rb

@@ -20,506 +20,35 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-require "stringio"
-
 module Omnizip
   module Implementations
     module SevenZip
       module LZMA2
-        # 7-Zip SDK LZMA2 encoder implementation.
-        #
-        # This encoder produces LZMA2 compressed data compatible with 7-Zip format.
-        # It uses the same LZMA encoding logic as XZ Utils, but with 7-Zip
-        # format requirements (no EOS marker, no padding).
+        # 7-Zip LZMA2 encoder.
         #
-        # Key differences from XZ Utils implementation:
-        # - No EOS marker (raw LZMA2 data ends with 0x00 control byte)
-        # - No chunk padding (XZ pads to 4-byte boundary)
-        # - No LZMA2 property byte in data stream (method ID only in container)
+        # The LZMA2 stream itself is format-identical to the raw LZMA2
+        # layer of the XZ Utils encoder (chunks + 0x00 end marker); the
+        # 7-Zip container carries the method properties, so no property
+        # byte is written into the data stream (standalone: false).
         #
-        # Based on LZMA SDK by Igor Pavlov
-        # Reference: https://www.7-zip.org/sdk.html
-        #
-        # LZMA2 format (as used by 7-Zip):
-        # - Control byte specifies chunk type and dictionary reset
-        # - Dictionary size follows in some chunk types
-        # - Uncompressed size follows in some chunk types
-        # - Compressed data follows
-        class Encoder < Base::LZMA2EncoderBase
-          include Omnizip::Algorithms::LZMA2Const
-
-          # Maximum chunk sizes (from LZMA2 specification)
-          MAX_UNCOMPRESSED_CHUNK = 2 * 1024 * 1024 # 2MB
-          MAX_COMPRESSED_CHUNK = 64 * 1024 # 64KB
-
-          # Encoding constants
-          UINT32_MAX = 0xFFFFFFFF
-          REPS = 4
-          MATCH_LEN_MIN = 2
-
-          attr_reader :dict_size, :lc, :lp, :pb, :standalone
-
-          # Initialize 7-Zip SDK LZMA2 encoder
-          #
-          # @param dict_size [Integer] Dictionary size (must be power of 2)
-          # @param lc [Integer] Literal context bits (0-8)
-          # @param lp [Integer] Literal position bits (0-4)
-          # @param pb [Integer] Position bits (0-4)
-          # @param standalone [Boolean] Include property byte (false for 7-Zip)
-          def initialize(dict_size:, lc: 3, lp: 0, pb: 2, standalone: false)
-            super
-
-            # Initialize shared state across all chunks
-            # Using XZ Utils components (tested and working)
-
-            @dictionary = Omnizip::Algorithms::LZMA::Dictionary.new(dict_size)
-            @state = Omnizip::Algorithms::LZMA::LZMAState.new(0)
-            @models = Omnizip::Algorithms::LZMA::XzProbabilityModels.new(lc,
-                                                                         lp, pb)
-            @match_finder = Omnizip::Algorithms::LZMA::MatchFinder.new(@dictionary)
-            @optimal = Omnizip::Algorithms::LZMA::OptimalEncoder.new(mode: :fast)
-
-            # Track previous byte for literal context
-            @prev_byte = 0
-
-            # First chunk always resets dictionary (7-Zip compatibility)
-            @need_dictionary_reset = true
-            @need_state_reset = false
-            @need_properties = true
-          end
-
-          # Encode data with LZMA2 compression
-          #
-          # @param data [String] Input data to compress
-          # @return [String] LZMA2 compressed data (7-Zip format)
-          def encode(data)
-            return "" if data.empty?
-
-            output = StringIO.new
-            output.set_encoding(Encoding::BINARY)
-
-            # Write property byte if standalone mode
-            if @standalone
-              prop_byte = encode_dict_size(@dict_size)
-              output.putc(prop_byte)
-            end
-
-            # Reset match finder state for each encoding session
-            @match_finder.reset
-
-            # Process in chunks
-            input = StringIO.new(data)
-            input.set_encoding(Encoding::BINARY)
-
-            while !input.eof?
-              chunk_data = input.read(MAX_UNCOMPRESSED_CHUNK)
-              break if chunk_data.nil? || chunk_data.empty?
-
-              chunk = encode_chunk(chunk_data)
-              output.write(chunk)
-
-              # Update reset flags for next chunk
-              @need_dictionary_reset = false
-              @need_state_reset = false
-              @need_properties = false
-            end
-
-            # End of stream marker (0x00)
-            output.write(Omnizip::Algorithms::LZMA2::LZMA2Chunk.end_chunk.to_bytes)
-
-            output.string
-          end
-
-          # Get implementation identifier
+        # This is a thin subclass rather than a copy: the XZ Utils
+        # encoder carries the fixes for global pos_state across
+        # chunks, announced state resets, the 64 KiB compressed-chunk
+        # cap (16-bit size field) and the range-encoder symbol-queue
+        # drain, which a duplicated implementation silently missed.
+        class Encoder < Implementations::XZUtils::LZMA2::Encoder
+          # 7-Zip callers pass the same option keys; only the property
+          # byte default differs (the container describes the coder).
+          def initialize(options = {})
+            super({ standalone: false }.merge(options))
+          end
+
+          # Get implementation identifier.
           #
           # @return [Symbol] :seven_zip_sdk
           def implementation_name
             :seven_zip_sdk
           end
-
-          private
-
-          # Encode a single chunk with LZMA2 compression
-          #
-          # Uses XZ Utils encoding logic (tested and compatible)
-          def encode_chunk(uncompressed_data)
-            compressed = try_compress(uncompressed_data)
-
-            # Decide: compressed vs uncompressed
-            # Use compressed if it's actually smaller
-            if compressed.bytesize >= uncompressed_data.bytesize
-              # Use uncompressed chunk
-              chunk = Omnizip::Algorithms::LZMA2::LZMA2Chunk.new(
-                chunk_type: :uncompressed,
-                uncompressed_data: uncompressed_data,
-                compressed_data: "",
-                need_dict_reset: @need_dictionary_reset,
-                need_state_reset: false,
-                need_props: false,
-              )
-              # After uncompressed chunk, next chunk needs state reset
-              @need_state_reset = true
-            else
-              # Use compressed chunk
-              chunk_properties = (((@pb * 5) + @lp) * 9) + @lc
-              chunk = Omnizip::Algorithms::LZMA2::LZMA2Chunk.new(
-                chunk_type: :compressed,
-                uncompressed_data: uncompressed_data,
-                compressed_data: compressed,
-                compressed_size: compressed.bytesize,
-                properties: chunk_properties,
-                need_dict_reset: @need_dictionary_reset,
-                need_state_reset: @need_state_reset,
-                need_props: true,
-              )
-            end
-
-            # Update dictionary with the chunk data
-            @dictionary.append(uncompressed_data)
-
-            # Update prev_byte for next chunk
-            if uncompressed_data.bytesize.positive?
-              @prev_byte = uncompressed_data.getbyte(uncompressed_data.bytesize - 1)
-            end
-
-            chunk.to_bytes
-          end
-
-          # Try to compress data using LZMA
-          #
-          # Uses XZ Utils encoding components (tested and working)
-          def try_compress(data)
-            # Create output buffer
-            output_buffer = StringIO.new
-            output_buffer.set_encoding(Encoding::BINARY)
-
-            # Create range encoder
-            encoder = Omnizip::Algorithms::LZMA::XzRangeEncoder.new(output_buffer)
-
-            # Feed all data to match finder first
-            @match_finder.feed(data)
-
-            # Initialize hash table
-            match_len_max = 2
-            end_pos = [
-              @dictionary.buffer.bytesize + data.bytesize - match_len_max, 0
-            ].max
-            @match_finder.skip(end_pos)
-
-            # Position in match finder's buffer for encoding
-            start_pos = @dictionary.buffer.bytesize
-            @current_start_pos = start_pos
-
-            pos = 0
-            while pos < data.bytesize
-              # Encode queued symbols if buffer getting full
-              if encoder.count > 20
-                encode_queued_symbols(encoder, output_buffer)
-              end
-
-              # Find matches at current position
-              match_pos = start_pos + pos
-              @match_finder.find_matches(match_pos)
-
-              # Get optimal encoding choice
-              distance, length = @optimal.find_optimal(
-                match_pos,
-                @match_finder,
-                @state,
-                @state.reps,
-                @models,
-              )
-
-              # Encode based on choice
-              if distance == UINT32_MAX || length == 1
-                encode_literal(data.getbyte(pos), encoder, pos)
-                pos += 1
-              elsif distance < REPS
-                encode_repeated_match(distance, length, encoder, pos, match_pos)
-                pos += length
-              else
-                actual_distance = distance - REPS
-                encode_match(actual_distance, length, encoder, pos, match_pos,
-                             data)
-                pos += length
-              end
-            end
-
-            # Flush encoder
-            encode_queued_symbols(encoder, output_buffer)
-            encoder.queue_flush
-            encode_queued_symbols(encoder, output_buffer)
-
-            output_buffer.string
-          end
-
-          # Encode queued symbols to output
-          def encode_queued_symbols(encoder, output)
-            return if encoder.none?
-
-            temp_buffer = "\0" * 10000
-            out_pos = Omnizip::Algorithms::LZMA::IntRef.new(0)
-
-            size_before = output.size
-
-            encoder.encode_symbols(temp_buffer, out_pos, 10000)
-
-            if out_pos.value.positive?
-              output.write(StringCompat.byteslice(temp_buffer, 0,
-                                                  out_pos.value))
-            end
-
-            output.size - size_before
-          end
-
-          # Compatibility helper for Ruby 3.0-3.1
-          module StringCompat
-            if "".respond_to?(:byteslice)
-              def self.byteslice(string, start, length)
-                string.byteslice(start, length)
-              end
-            else
-              def self.byteslice(string, start, length)
-                string.bytes[start, length]&.pack("C*") || ""
-              end
-            end
-          end
-
-          # Encode literal byte
-          def encode_literal(symbol, encoder, pos)
-            pos_state = pos & ((1 << @pb) - 1)
-
-            prob_is_match = @models.is_match[@state.value][pos_state]
-            encoder.queue_bit(prob_is_match, 0)
-
-            literal_offset = get_literal_state(pos, @prev_byte)
-            use_matched = @state.use_matched_literal?
-
-            @state.update_literal!
-
-            if use_matched
-              match_pos = @current_start_pos + pos
-              match_byte_pos = match_pos - @state.reps[0] - 1
-              match_byte = @match_finder.buffer.getbyte(match_byte_pos) if match_byte_pos >= 0 && match_byte_pos < @match_finder.buffer.bytesize
-
-              if match_byte.nil?
-                encode_normal_literal(literal_offset, symbol, encoder)
-              else
-                encode_matched_literal(literal_offset, match_byte, symbol,
-                                       encoder)
-              end
-            else
-              encode_normal_literal(literal_offset, symbol, encoder)
-            end
-
-            @prev_byte = symbol
-          end
-
-          # Encode normal match
-          def encode_match(distance, length, encoder, pos, match_pos,
-_input_data)
-            pos_state = pos & ((1 << @pb) - 1)
-
-            prob_is_match = @models.is_match[@state.value][pos_state]
-            encoder.queue_bit(prob_is_match, 1)
-
-            prob_is_rep = @models.is_rep[@state.value]
-            encoder.queue_bit(prob_is_rep, 0)
-
-            # Reps are stored 0-based (distance - 1), matching the decoder's
-            # rep0 convention.
-            @state.update_match!(distance - 1)
-
-            encode_match_length(length, pos_state, encoder)
-            encode_distance(distance, length, encoder)
-
-            last_byte_pos = match_pos - distance + length - 1
-            @prev_byte = @match_finder.buffer.getbyte(last_byte_pos) if last_byte_pos >= 0 && last_byte_pos < @match_finder.buffer.bytesize
-          end
-
-          # Encode repeated match
-          def encode_repeated_match(rep, length, encoder, pos, match_pos)
-            pos_state = pos & ((1 << @pb) - 1)
-
-            prob_is_match = @models.is_match[@state.value][pos_state]
-            encoder.queue_bit(prob_is_match, 1)
-
-            prob_is_rep = @models.is_rep[@state.value]
-            encoder.queue_bit(prob_is_rep, 1)
-
-            prob_is_rep0 = @models.is_rep0[@state.value]
-            if rep.zero?
-              encoder.queue_bit(prob_is_rep0, 0)
-
-              prob_is_rep0_long = @models.is_rep0_long[@state.value][pos_state]
-              encoder.queue_bit(prob_is_rep0_long, length == 1 ? 0 : 1)
-            else
-              encoder.queue_bit(prob_is_rep0, 1)
-
-              prob_is_rep1 = @models.is_rep1[@state.value]
-              if rep == 1
-                encoder.queue_bit(prob_is_rep1, 0)
-              else
-                encoder.queue_bit(prob_is_rep1, 1)
-
-                prob_is_rep2 = @models.is_rep2[@state.value]
-                encoder.queue_bit(prob_is_rep2, rep - 2)
-              end
-
-              # XZ Utils rep_match: capture the selected distance BEFORE
-              # rotating, then shift reps down.
-              distance = @state.reps[rep]
-              rep.downto(1) { |i| @state.reps[i] = @state.reps[i - 1] }
-              @state.reps[0] = distance
-            end
-
-            if length == 1
-              @state.update_short_rep!
-            else
-              # Rep matches use their own length models (decoder's
-              # rep_length_coder).
-              encode_match_length(length, pos_state, encoder,
-                                  @models.rep_len_encoder)
-              @state.update_long_rep!
-            end
-
-            # Last byte of the match region in the stream
-            last_byte_pos = match_pos + length - 1
-            @prev_byte = @match_finder.buffer.getbyte(last_byte_pos) if last_byte_pos >= 0 && last_byte_pos < @match_finder.buffer.bytesize
-          end
-
-          def get_literal_state(pos, prev_byte)
-            literal_mask = (0x100 << @lp) - (0x100 >> @lc)
-            # Factor of 3: each literal context owns a 0x300-sized subcoder
-            # (XZ Utils literal_subcoder macro); must match the decoder's
-            # base_offset = 3 * (lit_state << lc).
-            3 * ((((pos << 8) + prev_byte) & literal_mask) << @lc)
-          end
-
-          def encode_normal_literal(literal_offset, symbol, encoder)
-            context = 1
-            8.downto(1) do |i|
-              bit = (symbol >> (i - 1)) & 1
-              encoder.queue_bit(@models.literal[literal_offset + context], bit)
-              context = (context << 1) | bit
-            end
-          end
-
-          def encode_matched_literal(literal_offset, match_byte, symbol,
-encoder)
-            offset = 0x100
-            symbol += 0x100
-
-            while symbol < 0x10000
-              match_byte <<= 1
-              match_bit = match_byte & offset
-              subcoder_index = offset + match_bit + (symbol >> 8)
-              bit = (symbol >> 7) & 1
-
-              encoder.queue_bit(
-                @models.literal[literal_offset + subcoder_index], bit
-              )
-
-              symbol <<= 1
-              offset &= ~(match_byte ^ symbol)
-            end
-          end
-
-          # len_encoder selects the length model set: match_len_encoder for
-          # normal matches, rep_len_encoder for repeated matches.
-          def encode_match_length(length, pos_state, encoder,
-                                  len_encoder = @models.match_len_encoder)
-            len = length - 2
-
-            if len < 8
-              encoder.queue_bit(len_encoder.choice, 0)
-              encode_bittree(len_encoder.low[pos_state], 3, len, encoder)
-            elsif len < 16
-              encoder.queue_bit(len_encoder.choice, 1)
-              encoder.queue_bit(len_encoder.choice2, 0)
-              encode_bittree(len_encoder.mid[pos_state], 3, len - 8, encoder)
-            else
-              encoder.queue_bit(len_encoder.choice, 1)
-              encoder.queue_bit(len_encoder.choice2, 1)
-              encode_bittree(len_encoder.high, 8, len - 16, encoder)
-            end
-          end
-
-          def encode_distance(distance, length, encoder)
-            # get_dist_slot expects a 0-based distance (xz fastpos.h:
-            # dist 4 -> slot 4, dist 5 -> slot 4).
-            dist0 = distance - 1
-            dist_slot = get_dist_slot(dist0)
-            len_state = [length - 2, 3].min
-
-            encode_bittree(@models.dist_slot[len_state], 6, dist_slot, encoder)
-
-            if dist_slot >= 4
-              footer_bits = (dist_slot >> 1) - 1
-              base = (2 | (dist_slot & 1)) << footer_bits
-              dist_reduced = dist0 - base
-
-              if dist_slot < 14
-                encode_bittree_reverse(@models.dist_special, dist_reduced,
-                                       footer_bits, base - dist_slot - 1, encoder)
-              else
-                direct_bits = footer_bits - 4
-                encoder.queue_direct_bits(dist_reduced >> 4, direct_bits)
-                align_mask = (1 << 4) - 1
-                encode_bittree_reverse(@models.dist_align,
-                                       dist_reduced & align_mask, 4, 0, encoder)
-              end
-            end
-          end
-
-          def encode_bittree(probs, num_bits, value, encoder)
-            context = 1
-            num_bits.downto(1) do |i|
-              bit = (value >> (i - 1)) & 1
-              encoder.queue_bit(probs[context], bit)
-              context = (context << 1) | bit
-            end
-          end
-
-          def encode_bittree_reverse(probs, value, num_bits, offset, encoder)
-            context = 1
-            num_bits.times do |i|
-              bit = (value >> i) & 1
-              encoder.queue_bit(probs[offset + context], bit)
-              context = (context << 1) | bit
-            end
-          end
-
-          def get_dist_slot(distance)
-            if distance < 4
-              distance
-            else
-              slot = 0
-              dist = distance
-              while dist > 3
-                dist >>= 1
-                slot += 2
-              end
-              slot + dist
-            end
-          end
-
-          def encode_dict_size(dict_size)
-            d = [dict_size, DICT_SIZE_MIN].max
-
-            log2_size = 0
-            temp = d
-            while temp > 1
-              log2_size += 1
-              temp >>= 1
-            end
-
-            if d == (1 << log2_size)
-              [(log2_size - 12) * 2, 40].min
-            else
-              [((log2_size - 12) * 2) + 1, 40].min
-            end
-          end
         end
       end
     end

data/lib/omnizip/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Omnizip
-  VERSION = "0.3.18"
+  VERSION = "0.3.19"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omnizip
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.3.19
 platform: ruby
 authors:
 - Ribose Inc.