omnizip 0.4.7-aarch64-linux

data/lib/omnizip/checksums/crc_base.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2024 Ribose Inc.
+#
+# This file is part of Omnizip.
+#
+# Omnizip is a pure Ruby port of 7-Zip compression algorithms.
+# Based on the 7-Zip LZMA SDK by Igor Pavlov.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# See the COPYING file for the complete text of the license.
+#
+
+module Omnizip
+  module Checksums
+    # Abstract base class for CRC (Cyclic Redundancy Check) algorithms.
+    #
+    # This class provides the common framework for implementing CRC
+    # checksums using lookup tables for efficient computation. Subclasses
+    # must define their specific polynomial and bit width.
+    #
+    # The lookup table approach provides O(1) per-byte processing time,
+    # making it significantly faster than bit-by-bit calculation.
+    #
+    # Architecture:
+    # - Pre-computed lookup tables stored as class constants
+    # - Incremental computation support via update method
+    # - Initial and final XOR values for standard CRC variants
+    #
+    # @abstract Subclass and override polynomial-specific methods
+    class CrcBase
+      # @return [Integer] current CRC value
+      attr_reader :value
+
+      # Initialize a new CRC calculator with default initial value.
+      #
+      # The initial value is typically all 1s (inverted 0) to detect
+      # leading zeros in the input data.
+      def initialize
+        @value = initial_value
+      end
+
+      # Update the CRC value with new data.
+      #
+      # This method processes the input data byte-by-byte using the
+      # lookup table, allowing incremental CRC calculation.
+      #
+      # @param data [String] binary string to process
+      # @return [self] for method chaining
+      def update(data)
+        data.each_byte do |byte|
+          @value = process_byte(@value, byte)
+        end
+        self
+      end
+
+      # Reset the CRC calculator to initial state.
+      #
+      # @return [self] for method chaining
+      def reset
+        @value = initial_value
+        self
+      end
+
+      # Get the finalized CRC value.
+      #
+      # Applies the final XOR operation required by most CRC standards
+      # to produce the final checksum value.
+      #
+      # @return [Integer] final CRC checksum
+      def finalize
+        @value ^ final_xor
+      end
+
+      # Calculate CRC for data in one operation.
+      #
+      # This is a convenience method that combines initialize, update,
+      # and finalize into a single call.
+      #
+      # @param data [String] binary string to checksum
+      # @return [Integer] final CRC checksum
+      def self.calculate(data)
+        new.update(data).finalize
+      end
+
+      protected
+
+      # Process a single byte through the CRC algorithm.
+      #
+      # This method must be implemented by subclasses to perform the
+      # actual CRC calculation using the lookup table.
+      #
+      # @param crc [Integer] current CRC value
+      # @param byte [Integer] byte to process (0-255)
+      # @return [Integer] updated CRC value
+      # @abstract
+      def process_byte(crc, byte)
+        raise NotImplementedError,
+              "#{self.class} must implement #process_byte"
+      end
+
+      # Get the initial CRC value.
+      #
+      # @return [Integer] initial value (typically all 1s)
+      # @abstract
+      def initial_value
+        raise NotImplementedError,
+              "#{self.class} must implement #initial_value"
+      end
+
+      # Get the final XOR value.
+      #
+      # @return [Integer] value to XOR with final result
+      # @abstract
+      def final_xor
+        raise NotImplementedError,
+              "#{self.class} must implement #final_xor"
+      end
+
+      # Get the lookup table for this CRC variant.
+      #
+      # @return [Array<Integer>] 256-entry lookup table
+      # @abstract
+      def self.lookup_table
+        raise NotImplementedError,
+              "#{self} must implement .lookup_table"
+      end
+
+      # Generate a CRC lookup table for given polynomial.
+      #
+      # This method pre-computes all possible CRC values for single-byte
+      # inputs, enabling efficient O(1) per-byte processing.
+      #
+      # @param polynomial [Integer] CRC polynomial
+      # @param bits [Integer] bit width (32 or 64)
+      # @return [Array<Integer>] 256-entry lookup table
+      def self.generate_table(polynomial, bits)
+        mask = (1 << bits) - 1
+        (0..255).map do |i|
+          crc = i
+          8.times do
+            if crc.anybits?(1)
+              crc = (crc >> 1) ^ polynomial
+            else
+              crc >>= 1
+            end
+          end
+          crc & mask
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/checksums/registration.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# Checksum registration - triggers autoload and registers all checksums
+# This file should be required after checksums.rb which sets up autoloads
+
+module Omnizip
+  module Checksums
+    # Touch constants to trigger autoload
+    Crc32
+    Crc64
+  end
+end
+
+# Register checksum algorithms
+Omnizip::ChecksumRegistry.register(:crc32, Omnizip::Checksums::Crc32)
+Omnizip::ChecksumRegistry.register(:crc64, Omnizip::Checksums::Crc64)

data/lib/omnizip/checksums/verifier.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2025 Ribose Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+require "zlib"
+require "digest"
+
+module Omnizip
+  module Checksums
+    # Checksum verification utilities for XZ format
+    #
+    # XZ supports multiple check types:
+    # - 0: None (no checksum)
+    # - 1: CRC32 (4 bytes)
+    # - 4: CRC64 (8 bytes)
+    # - 10: SHA256 (32 bytes)
+    class Verifier
+      # Check type constants (from XZ spec)
+      CHECK_NONE = 0
+      CHECK_CRC32 = 1
+      CHECK_CRC64 = 4
+      CHECK_SHA256 = 10
+
+      # Check sizes in bytes
+      CHECK_SIZES = {
+        CHECK_NONE => 0,
+        CHECK_CRC32 => 4,
+        CHECK_CRC64 => 8,
+        CHECK_SHA256 => 32,
+      }.freeze
+
+      # Verify CRC32 checksum
+      #
+      # @param data [String] Data to verify
+      # @param expected [Integer] Expected CRC32 value
+      # @return [Boolean] True if checksum matches
+      def self.verify_crc32(data, expected)
+        Zlib.crc32(data) == expected
+      end
+
+      # Verify CRC64 checksum
+      #
+      # @param data [String] Data to verify
+      # @param expected [Integer] Expected CRC64 value
+      # @return [Boolean] True if checksum matches
+      def self.verify_crc64(data, expected)
+        Crc64.calculate(data) == expected
+      end
+
+      # Verify SHA256 checksum
+      #
+      # @param data [String] Data to verify
+      # @param expected [String] Expected SHA256 digest (binary, 32 bytes)
+      # @return [Boolean] True if checksum matches
+      def self.verify_sha256(data, expected)
+        Digest::SHA256.digest(data) == expected
+      end
+
+      # Generic verify based on check type
+      #
+      # @param data [String] Data to verify
+      # @param expected [String] Expected checksum (binary format)
+      # @param check_type [Integer] Check type (0=None, 1=CRC32, 4=CRC64, 10=SHA256)
+      # @return [Boolean] True if checksum matches or check type is None
+      # @raise [ArgumentError] If check type is unknown
+      def self.verify(data, expected, check_type)
+        case check_type
+        when CHECK_NONE
+          true # No checksum to verify
+        when CHECK_CRC32
+          expected_crc = expected.unpack1("V")
+          verify_crc32(data, expected_crc)
+        when CHECK_CRC64
+          expected_crc = expected.unpack1("Q<")
+          verify_crc64(data, expected_crc)
+        when CHECK_SHA256
+          verify_sha256(data, expected)
+        else
+          raise Omnizip::UnknownChecksumError, "Unknown check type: #{check_type}"
+        end
+      end
+
+      # Get check size in bytes
+      #
+      # @param check_type [Integer] Check type
+      # @return [Integer] Size of checksum in bytes
+      def self.check_size(check_type)
+        CHECK_SIZES.fetch(check_type, 0)
+      end
+
+      # Calculate checksum for data based on check type
+      #
+      # @param data [String] Data to checksum
+      # @param check_type [Integer] Check type
+      # @return [String] Checksum in binary format
+      def self.calculate(data, check_type)
+        case check_type
+        when CHECK_NONE
+          ""
+        when CHECK_CRC32
+          [Zlib.crc32(data)].pack("V")
+        when CHECK_CRC64
+          [Crc64.calculate(data)].pack("Q<")
+        when CHECK_SHA256
+          Digest::SHA256.digest(data)
+        else
+          raise Omnizip::UnknownChecksumError, "Unknown check type: #{check_type}"
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/checksums.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Omnizip
+  # Checksum implementations
+  module Checksums
+    autoload :CrcBase, "omnizip/checksums/crc_base"
+    autoload :Crc32, "omnizip/checksums/crc32"
+    autoload :Crc64, "omnizip/checksums/crc64"
+    autoload :Verifier, "omnizip/checksums/verifier"
+  end
+end

data/lib/omnizip/chunked/memory_manager.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "tempfile"
+
+module Omnizip
+  module Chunked
+    # Manage memory usage and enforce limits during chunked operations
+    class MemoryManager
+      DEFAULT_MAX_MEMORY = 256 * 1024 * 1024 # 256MB
+
+      attr_reader :current_usage, :max_memory
+
+      # Initialize memory manager
+      # @param options [Hash] Manager options
+      # @option options [Integer] :max Maximum memory in bytes
+      # @option options [String] :temp_dir Temporary directory for spill files
+      # @option options [Symbol] :strategy Spill strategy (:disk or :error)
+      def initialize(max: DEFAULT_MAX_MEMORY, temp_dir: nil, strategy: :disk)
+        @max_memory = max
+        @current_usage = 0
+        @buffers = {} # buffer => size mapping
+        @temp_files = []
+        @temp_dir = temp_dir
+        @strategy = strategy
+      end
+
+      # Allocate memory or spill to disk
+      # @param size [Integer] Size to allocate in bytes
+      # @return [String, Tempfile] Buffer or temp file
+      def allocate(size)
+        if can_allocate_in_memory?(size)
+          allocate_buffer(size)
+        else
+          handle_overflow(size)
+        end
+      end
+
+      # Release allocated memory or temp file
+      # @param buffer [String, Tempfile] Buffer to release
+      # @return [Integer] Bytes released
+      def release(buffer)
+        case buffer
+        when String
+          release_buffer(buffer)
+        when Tempfile, File
+          release_temp_file(buffer)
+        else
+          0
+        end
+      end
+
+      # Get available memory
+      # @return [Integer] Available memory in bytes
+      def available
+        [@max_memory - @current_usage, 0].max
+      end
+
+      # Check if over memory limit
+      # @return [Boolean] True if over limit
+      def over_limit?
+        @current_usage > @max_memory
+      end
+
+      # Get memory usage ratio
+      # @return [Float] Usage from 0.0 to 1.0
+      def usage_ratio
+        return 0.0 if @max_memory.zero?
+
+        @current_usage.to_f / @max_memory
+      end
+
+      # Cleanup all resources
+      # @return [self]
+      def cleanup
+        @buffers.clear
+        @temp_files.each do |tf|
+          begin
+            tf.close
+          rescue StandardError
+            nil
+          end
+          begin
+            tf.unlink
+          rescue StandardError
+            nil
+          end
+        end
+        @temp_files.clear
+        @current_usage = 0
+        self
+      end
+
+      # Execute block with automatic cleanup
+      # @yield [manager] Block to execute with manager
+      # @return [Object] Block result
+      def self.with_manager(**options)
+        manager = new(**options)
+        begin
+          yield manager
+        ensure
+          manager.cleanup
+        end
+      end
+
+      # Spill buffer to disk
+      # @param buffer [String] Buffer data to spill
+      # @return [Tempfile] Temporary file
+      def spill_to_disk(buffer = nil)
+        tf = create_temp_file
+        tf.write(buffer) if buffer
+        tf.rewind
+        @temp_files << tf
+        tf
+      end
+
+      private
+
+      # Check if size can be allocated in memory
+      def can_allocate_in_memory?(size)
+        @current_usage + size <= @max_memory
+      end
+
+      # Allocate in-memory buffer
+      def allocate_buffer(size)
+        buffer = String.new(capacity: size, encoding: Encoding::BINARY)
+        @buffers[buffer] = size
+        @current_usage += size
+        buffer
+      end
+
+      # Release in-memory buffer
+      def release_buffer(buffer)
+        size = @buffers.delete(buffer)
+        if size
+          @current_usage -= size
+          size
+        else
+          0
+        end
+      end
+
+      # Release temp file
+      def release_temp_file(file)
+        if @temp_files.delete(file)
+          size = begin
+            file.size
+          rescue StandardError
+            0
+          end
+          begin
+            file.close
+          rescue StandardError
+            nil
+          end
+          begin
+            file.unlink
+          rescue StandardError
+            nil
+          end
+          size
+        else
+          0
+        end
+      end
+
+      # Handle memory overflow based on strategy
+      def handle_overflow(size)
+        case @strategy
+        when :disk
+          spill_to_disk
+        when :error
+          message = "Memory limit exceeded: " \
+                    "#{@current_usage + size} > #{@max_memory}"
+          raise MemoryError, message
+        else
+          raise ArgumentError, "Unknown spill strategy: #{@strategy}"
+        end
+      end
+
+      # Create temporary file
+      def create_temp_file
+        Tempfile.new(
+          ["omnizip_chunk_", ".tmp"],
+          @temp_dir,
+          binmode: true,
+        )
+      end
+    end
+
+    # Error raised when memory limit is exceeded
+    class MemoryError < Omnizip::Error
+    end
+  end
+end

data/lib/omnizip/chunked/reader.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Chunked
+    # Read large files in chunks for memory-efficient processing
+    class Reader
+      DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024 # 64MB
+
+      attr_reader :file_path, :chunk_size, :total_size
+
+      # Initialize a chunked reader
+      # @param file_path [String] Path to file to read
+      # @param options [Hash] Reader options
+      # @option options [Integer] :chunk_size Chunk size in bytes
+      def initialize(file_path, chunk_size: DEFAULT_CHUNK_SIZE)
+        @file_path = file_path
+        @chunk_size = chunk_size
+        @total_size = File.size(file_path)
+        @position = 0
+      end
+
+      # Read next chunk from file
+      # @return [String, nil] Chunk data or nil if EOF
+      def read_chunk
+        return nil if @position >= @total_size
+
+        File.open(@file_path, "rb") do |f|
+          f.seek(@position)
+          chunk = f.read(@chunk_size)
+          @position += chunk.bytesize if chunk
+          chunk
+        end
+      end
+
+      # Iterate through all chunks
+      # @yield [chunk, position, total] Block called for each chunk
+      def each_chunk
+        reset
+        while (chunk = read_chunk)
+          yield chunk, @position - chunk.bytesize, @total_size
+        end
+      end
+
+      # Get current progress as percentage
+      # @return [Float] Progress from 0.0 to 1.0
+      def progress
+        return 1.0 if @total_size.zero?
+
+        @position.to_f / @total_size
+      end
+
+      # Reset reader to beginning of file
+      # @return [self]
+      def reset
+        @position = 0
+        self
+      end
+
+      # Check if at end of file
+      # @return [Boolean] True if at EOF
+      def eof?
+        @position >= @total_size
+      end
+
+      # Get number of chunks needed for file
+      # @return [Integer] Number of chunks
+      def chunk_count
+        (@total_size.to_f / @chunk_size).ceil
+      end
+
+      # Get remaining bytes to read
+      # @return [Integer] Remaining bytes
+      def remaining
+        @total_size - @position
+      end
+    end
+  end
+end

data/lib/omnizip/chunked/writer.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "stringio"
+
+module Omnizip
+  module Chunked
+    # Write large files incrementally in chunks
+    class Writer
+      DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024 # 64MB
+      FLUSH_THRESHOLD = 10 # Flush every N chunks
+
+      attr_reader :output_path, :chunk_size, :written
+
+      # Initialize a chunked writer
+      # @param output_path [String] Path to output file
+      # @param options [Hash] Writer options
+      # @option options [Integer] :chunk_size Chunk size in bytes
+      def initialize(output_path, chunk_size: DEFAULT_CHUNK_SIZE)
+        @output_path = output_path
+        @chunk_size = chunk_size
+        @written = 0
+        @chunks_written = 0
+        @file_handle = nil
+      end
+
+      # Write a chunk to file
+      # @param chunk [String] Data to write
+      # @return [Integer] Bytes written
+      def write_chunk(chunk)
+        ensure_file_open
+
+        bytes = @file_handle.write(chunk)
+        @written += bytes
+        @chunks_written += 1
+
+        # Flush periodically to disk
+        flush if (@chunks_written % FLUSH_THRESHOLD).zero?
+
+        bytes
+      end
+
+      # Write data from a source in chunks
+      # @param source [String, IO] Source to read from
+      # @return [Integer] Total bytes written
+      def write_from(source)
+        case source
+        when String
+          # File path
+          if File.exist?(source)
+            Reader.new(source, chunk_size: @chunk_size).each_chunk do |chunk|
+              write_chunk(chunk)
+            end
+          else
+            # Treat as data
+            write_chunk(source)
+          end
+        when IO, StringIO
+          # IO object
+          while (chunk = source.read(@chunk_size))
+            break if chunk.empty?
+
+            write_chunk(chunk)
+          end
+        else
+          raise ArgumentError, "Unsupported source type: #{source.class}"
+        end
+
+        @written
+      end
+
+      # Flush buffered data to disk
+      # @return [self]
+      def flush
+        @file_handle&.flush
+        self
+      end
+
+      # Close the file handle
+      # @return [self]
+      def close
+        if @file_handle
+          flush
+          @file_handle.close
+          @file_handle = nil
+        end
+        self
+      end
+
+      # Finalize the file (alias for close)
+      # @return [self]
+      def finish
+        close
+      end
+
+      # Execute block with writer, auto-close
+      # @yield [writer] Block to write data
+      # @return [Integer] Total bytes written
+      def self.with_writer(output_path, **options)
+        writer = new(output_path, **options)
+        begin
+          yield writer
+          writer.written
+        ensure
+          writer.close
+        end
+      end
+
+      private
+
+      # Ensure file handle is open
+      def ensure_file_open
+        return if @file_handle
+
+        # Create parent directory if needed
+        dir = File.dirname(@output_path)
+        FileUtils.mkdir_p(dir)
+
+        @file_handle = File.open(@output_path, "wb")
+      end
+    end
+  end
+end