omnizip 0.3.37 → 0.3.38

data/lib/omnizip/formats/rar/rar5/compression/lzss.rb

@@ -42,9 +42,11 @@ module Omnizip
               #
               # @return [Boolean] true if implemented
               def available?
-                # Full LZSS decoder is now implemented
-                # Encoder is not yet compatible with official RAR tools
-                true
+                # Decoder is implemented; the encoder is NOT compatible
+                # with official RAR tools (unrar "tests" its output as
+                # OK while extracting empty files). Callers fall back
+                # to STORE until a RAR-compatible encoder lands.
+                false
               end
 
               # Compress data using RAR5 LZSS

data/lib/omnizip/formats/rar/rar5/header.rb

@@ -89,9 +89,18 @@ header_data: "")
 
         # Main archive header
         class MainHeader < Header
-          def initialize(flags: 0)
-            # Main header has no data
-            super(HEADER_TYPE_MAIN, flags: flags)
+          # Archive flags: 0x0001 volume, 0x0002 volume number
+          # present, 0x0004 solid, 0x0008 recovery, 0x0010 locked
+          ARCHIVE_FLAG_VOLUME = 0x0001
+          ARCHIVE_FLAG_VOLUME_NUMBER = 0x0002
+
+          def initialize(archive_flags: 0, volume_number: nil)
+            data = VINT.encode(archive_flags)
+            if archive_flags.anybits?(ARCHIVE_FLAG_VOLUME_NUMBER)
+              data.concat(VINT.encode(volume_number.to_i))
+            end
+
+            super(HEADER_TYPE_MAIN, header_data: data.pack("C*"))
           end
         end
 
@@ -102,69 +111,73 @@ header_data: "")
           FILE_HAS_MTIME = 0x0002
           FILE_HAS_CRC32 = 0x0004
 
+          # Compression information field layout (RAR 5.0 spec):
+          # bits 0-5 algorithm version (0), bit 7 solid,
+          # bits 8-10 method (0=store, 1-5=LZMA), bits 11-15
+          # dictionary size (128 KB * 2^N).
+          DICT_BASE = 128 * 1024
+
           def initialize(filename:, file_size:, compressed_size:,
-compression_method: 0, flags: 0, mtime: nil, crc32: nil, extra_area: nil)
+                         compression_method: 0, dict_size: 0,
+                         flags: 0, mtime: nil, crc32: nil, extra_area: nil)
             # Build file flags based on what's provided
             file_flags = 0
             file_flags |= FILE_HAS_MTIME if mtime
             file_flags |= FILE_HAS_CRC32 if crc32
 
             # Build header data with file information
-            data = build_file_data(filename, file_size, compressed_size,
-                                   file_flags, compression_method, mtime, crc32)
+            data = build_file_data(filename, file_size,
+                                   compression_method, dict_size,
+                                   file_flags, mtime, crc32)
             super(HEADER_TYPE_FILE, flags: flags, data_area_size: compressed_size, header_data: data, extra_area: extra_area)
           end
 
           private
 
-          def build_file_data(filename, file_size, _compressed_size,
-file_flags, compression_method, mtime, crc32)
+          def build_file_data(filename, file_size, compression_method,
+                              dict_size, file_flags, mtime, crc32)
             data = []
 
             # File flags (VINT)
             data.concat(VINT.encode(file_flags))
 
-            # Unp size (uncompressed size, VINT)
+            # Unpacked size (VINT)
             data.concat(VINT.encode(file_size))
 
-            # Attributes (VINT) - ALWAYS present in RAR5
-            # Use 0x2483 from official RAR (standard regular file with correct permissions)
-            data.concat(VINT.encode(0x2483))
-
-            # Mystery VINT with value 0x02 - observed in official RAR
-            # This appears after attributes in official archives - ALWAYS present
-            data.concat(VINT.encode(0x02))
+            # Attributes (VINT) - ALWAYS present in RAR5.
+            # Unix host: standard regular-file mode 0o100644.
+            data.concat(VINT.encode(0o100644))
 
             # mtime (optional) - only if FILE_HAS_MTIME flag is set
-            # RAR5 stores mtime as Unix timestamp (seconds since epoch) in DOS format
+            # RAR5 stores mtime as Unix timestamp (seconds since epoch)
             # Format: 4 bytes little-endian (NOT a VINT)
             if file_flags.anybits?(FILE_HAS_MTIME) && mtime
-              # Convert Time to Unix timestamp (seconds since 1970-01-01 00:00:00 UTC)
-              unix_time = mtime.to_i
-              # Pack as 32-bit unsigned little-endian
-              data.concat([unix_time].pack("V").bytes)
+              data.concat([mtime.to_i].pack("V").bytes)
             end
 
             # Data CRC32 (optional) - only if FILE_HAS_CRC32 flag is set
             # Format: 4 bytes little-endian (NOT a VINT)
             if file_flags.anybits?(FILE_HAS_CRC32) && crc32
-              # Pack as 32-bit unsigned little-endian
               data.concat([crc32].pack("V").bytes)
             end
 
-            # Compression info (VINT)
-            # Bits 0-5: method (0=STORE, 1-5=LZMA with different levels)
-            # Bits 6+: version
-            data.concat(VINT.encode(compression_method))
+            # Compression information (VINT): version 0 in bits 0-5,
+            # method in bits 8-10, dictionary size in bits 11-15
+            # (128 KB * 2^N).
+            dict_bits = if dict_size.positive?
+                          Math.log2(dict_size / DICT_BASE).round.clamp(0, 15)
+                        else
+                          0
+                        end
+            compression_info = (compression_method << 8) | (dict_bits << 11)
+            data.concat(VINT.encode(compression_info))
 
             # Host OS (VINT) - 1 = Unix
-            data.concat(VINT.encode(1)) # Unix
+            data.concat(VINT.encode(1))
 
-            # Name length (VINT)
+            # Name length (VINT) and name
             name_bytes = filename.encode("UTF-8").bytes
             data.concat(VINT.encode(name_bytes.size))
-
-            # Name
             data.concat(name_bytes)
 
             data.pack("C*")
@@ -173,9 +186,13 @@ file_flags, compression_method, mtime, crc32)
 
         # End of archive header
         class EndHeader < Header
-          def initialize
-            # End header is minimal
-            super(HEADER_TYPE_END, flags: 0)
+          # End of archive flags: 0x0001 volume and not the last
+          # volume in the set
+          END_FLAG_VOLUME_NEXT = 0x0001
+
+          def initialize(end_flags: 0)
+            super(HEADER_TYPE_END,
+                  header_data: VINT.encode(end_flags).pack("C*"))
           end
         end
       end

data/lib/omnizip/formats/rar/rar5/multi_volume/volume_writer.rb

@@ -18,11 +18,10 @@ module Omnizip
           #   writer.write_end_header
           #   writer.close
           class VolumeWriter
-            # Main header flags for volume archives
-            # NOTE: Bits 0-1 are reserved for common flags (EXTRA_AREA, DATA_AREA)
-            # Use bits 2+ for format-specific flags
-            VOLUME_ARCHIVE_FLAG = 0x0004  # Bit 2: This is a volume archive
-            VOLUME_NUMBER_FLAG  = 0x0008  # Bit 3: Volume number present in extra area
+            # Main ARCHIVE flags (spec): 0x0001 volume,
+            # 0x0002 volume number field present
+            VOLUME_ARCHIVE_FLAG = MainHeader::ARCHIVE_FLAG_VOLUME
+            VOLUME_NUMBER_FLAG  = MainHeader::ARCHIVE_FLAG_VOLUME_NUMBER
 
             # End header flags
             VOLUME_END_FLAG = 0x0001 # Not the last volume (more volumes follow)
@@ -95,26 +94,18 @@ module Omnizip
             def write_main_header
               raise "Volume not open" unless @io
 
-              # Set volume-specific flags
-              flags = VOLUME_ARCHIVE_FLAG
-
-              # Add volume number in extra area for volumes 2+
-              nil
+              # Archive flags: every volume declares 0x0001; volumes
+              # after the first also carry the volume number field
+              # (spec: number is N-1, absent on the first volume).
+              archive_flags = VOLUME_ARCHIVE_FLAG
+              volume_number = nil
               if @volume_number > 1
-                flags |= VOLUME_NUMBER_FLAG
-                # Volume number as VINT in extra area
-                VINT.encode(@volume_number).pack("C*")
+                archive_flags |= VOLUME_NUMBER_FLAG
+                volume_number = @volume_number - 1
               end
 
-              header = MainHeader.new(flags: flags)
-
-              # Extra areas are not written: MainHeader does not
-              # model them and the omnizip-rs reference implements
-              # RAR5 reading only, so there is no writer-side spec
-              # to port. Volumes work without extras (the field is
-              # optional).
-
-              @io.write(header.encode)
+              @io.write(MainHeader.new(archive_flags: archive_flags,
+                                       volume_number: volume_number).encode)
             end
 
             # Write file data (header + compressed data)
@@ -135,18 +126,10 @@ module Omnizip
             def write_end_header
               raise "Volume not open" unless @io
 
-              # Set END_OF_ARCHIVE flags
-              flags = 0
-              flags | VOLUME_END_FLAG unless @is_last
-
-              # EndHeader always encodes the plain END_OF_ARCHIVE
-              # marker: the RAR5 spec makes the volume flag optional,
-              # readers position the marker inside the volume
-              # sequence, and the omnizip-rs reference (read-side)
-              # carries no writer-side volume-flag encoding to port.
-              header = EndHeader.new
-
-              @io.write(header.encode)
+              # End-of-archive flags: 0x0001 marks a volume with more
+              # volumes following; the last volume ends with 0.
+              end_flags = @is_last ? 0 : VOLUME_END_FLAG
+              @io.write(EndHeader.new(end_flags: end_flags).encode)
             end
 
             # Generate volume filename from base name

data/lib/omnizip/formats/rar/rar5/vint.rb

@@ -6,57 +6,45 @@ module Omnizip
       module Rar5
         # Variable-length integer encoding/decoding for RAR5 format
         module VINT
-          # Encode integer as VINT bytes
-          # @param value [Integer] Value to encode (0 to 2^62)
+          # Encode integer as VINT bytes per the RAR 5.0 spec: one or
+          # more bytes, each carrying 7 data bits starting with the
+          # least significant group; the high bit of every byte but
+          # the last is the continuation flag.
+          #
+          # @param value [Integer] Value to encode (0 to 2^63)
           # @return [Array<Integer>] VINT bytes
           def self.encode(value)
-            return [value] if value < 0x80
+            raise ArgumentError, "VINT cannot encode negative values" if value.negative?
 
             bytes = []
-            # Determine byte count needed
-            byte_count = 1
-            test_value = value
-            while test_value >= (1 << (7 * byte_count))
-              byte_count += 1
-            end
-
-            # First byte: continuation bits + high bits
-            first_byte = (0xFF << (9 - byte_count)) & 0xFF
-            first_byte |= (value >> (8 * (byte_count - 1))) & 0x7F
-            bytes << first_byte
-
-            # Remaining bytes
-            (byte_count - 1).downto(1) do |i|
-              bytes << ((value >> (8 * (i - 1))) & 0xFF)
+            loop do
+              byte = value & 0x7F
+              value >>= 7
+              byte |= 0x80 if value.positive?
+              bytes << byte
+              break if value.zero?
             end
 
             bytes
           end
 
-          # Decode VINT from IO stream
+          # Decode VINT from IO stream (spec encoding, mirroring
+          # BlockParser#read_vint)
+          #
           # @param io [IO] Input stream
           # @return [Integer] Decoded value
           def self.decode(io)
-            first_byte = io.readbyte
-            return first_byte if first_byte < 0x80
-
-            # Count continuation bits
-            byte_count = 0
-            mask = 0x80
-            while first_byte.anybits?(mask)
-              byte_count += 1
-              mask >>= 1
-            end
-
-            # Extract value from first byte
-            value = first_byte & (0xFF >> (byte_count + 1))
-
-            # Read remaining bytes
-            byte_count.times do
-              value = (value << 8) | io.readbyte
+            result = 0
+            shift = 0
+            loop do
+              byte = io.readbyte
+              result |= (byte & 0x7F) << shift
+              break if byte.nobits?(0x80)
+
+              shift += 7
             end
 
-            value
+            result
           end
         end
       end

data/lib/omnizip/formats/rar/rar5/writer.rb

@@ -177,7 +177,10 @@ module Omnizip
               write_signature(f)
               write_main_header(f)
 
-              if @options[:solid]
+              # Solid mode needs a RAR-compatible LZSS encoder; until
+              # one lands, fall back to independent STORE entries so
+              # the archive is actually extractable by unrar.
+              if @options[:solid] && Compression::Lzss.available?
                 write_solid_block(f, @files)
               else
                 @files.each do |file|
@@ -251,6 +254,7 @@ module Omnizip
                 file_size: file_info[:size],
                 compressed_size: (idx.zero? ? result[:compressed_size] : 0), # Only first file has compressed size
                 compression_method: compression_method,
+                dict_size: dictionary_size_for(compression_method),
                 mtime: @options[:include_mtime] ? file[:mtime] : nil,
                 crc32: nil, # No CRC32 for solid/LZMA
               )
@@ -307,12 +311,13 @@ module Omnizip
             if @encryption_manager
               encryption_result = @encryption_manager.encrypt_file_data(compressed_data)
               final_data = encryption_result[:encrypted_data]
-              # The generated EncryptionHeader (salt/IV) is not
-              # persisted into the archive here, so archives this
-              # path writes cannot be decrypted afterwards; the RAR5
-              # CRYPT extra record has no write-side reference to
-              # port (omnizip-rar implements reading only).
-              # Encryption remains read-verified only.
+              # Persist the salt/IV as the RAR5 file encryption extra
+              # record (spec type 0x01) — without it the archive
+              # cannot be decrypted by anything.
+              enc_record = build_encryption_extra_record(
+                encryption_result[:header],
+              )
+              extra_area = extra_area ? extra_area + enc_record : enc_record
             else
               final_data = compressed_data
             end
@@ -332,6 +337,7 @@ module Omnizip
               file_size: data.bytesize,
               compressed_size: final_data.bytesize,
               compression_method: compression_method,
+              dict_size: dictionary_size_for(compression_method),
               mtime: @options[:include_mtime] ? file[:mtime] : nil,
               crc32: file_crc32,
               extra_area: extra_area,
@@ -356,6 +362,22 @@ module Omnizip
           #
           # @param data [String] File data
           # @return [Integer] Compression method ID
+          # Dictionary size matching the compression modules' level
+          # table (128 KB << N), for the file header's dictionary
+          # bits. STORE needs no dictionary.
+          def dictionary_size_for(method)
+            return 0 if method == Compression::Store::METHOD
+
+            1 << case method
+                 when 1 then 18 # 256 KB
+                 when 2 then 20 # 1 MB
+                 when 3 then 22 # 4 MB
+                 when 4 then 23 # 8 MB
+                 when 5 then 24 # 16 MB
+                 else 22
+                 end
+          end
+
           def select_compression_method(data)
             case @options[:compression]
             when :store
@@ -367,7 +389,9 @@ module Omnizip
               if Compression::Lzss.available?
                 Compression::Lzss.method_id(level)
               else
-                # LZSS not implemented, fall back to STORE
+                warn "omnizip: RAR5 #{@options[:compression] == :lzma ? 'lzma' : 'lzss'} " \
+                     "compression is not interoperable yet; storing data " \
+                     "uncompressed instead"
                 Compression::Store::METHOD
               end
             when :auto
@@ -418,15 +442,27 @@ module Omnizip
           def build_compression_extra_area(properties)
             return nil if properties.nil?
 
-            extra_data = []
+            # Extra record format per the RAR 5.0 spec: Size(vint),
+            # Type(vint), Data. Type 0x03 carries the compression
+            # parameters.
+            record = VINT.encode(0x03) + properties.bytes
+            VINT.encode(record.length) + record
+          end
 
-            # Extra record type: 0x03 = Compression parameters
-            extra_data.concat(VINT.encode(0x03))
+          # File encryption extra record (spec type 0x01): AES-256
+          # parameters the reader needs to derive the key and IV.
+          def build_encryption_extra_record(header)
+            require "base64"
 
-            # Compression properties (method-specific)
-            extra_data.concat(properties.bytes)
+            kdf_count = Math.log2(header.kdf_iterations).round
+            record = [].concat(VINT.encode(0x01)) # record type
+              .concat(VINT.encode(header.version)) # 0 = AES-256
+              .concat(VINT.encode(0)) # flags: no check data
+            record << kdf_count # 1 byte, log2
+            record.concat(Base64.strict_decode64(header.salt).bytes)
+            record.concat(Base64.strict_decode64(header.iv).bytes)
 
-            extra_data.pack("C*")
+            (VINT.encode(record.length) + record).pack("C*")
           end
 
           # Generate PAR2 recovery files for archives

data/lib/omnizip/formats/rar.rb

@@ -37,10 +37,12 @@ module Omnizip
         autoload :VINT, "omnizip/formats/rar/rar5/vint"
         autoload :CRC32, "omnizip/formats/rar/rar5/crc32"
         autoload :Header, "omnizip/formats/rar/rar5/header"
+        # MainHeader/FileHeader live in header.rb directly under Rar5;
+        # without their own autoload entries, referencing them only
+        # works if something else loaded the file first.
+        autoload :MainHeader, "omnizip/formats/rar/rar5/header"
+        autoload :FileHeader, "omnizip/formats/rar/rar5/header"
         autoload :Writer, "omnizip/formats/rar/rar5/writer"
-        autoload :Reader, "omnizip/formats/rar/rar5/reader"
-        autoload :Compressor, "omnizip/formats/rar/rar5/compressor"
-        autoload :Decompressor, "omnizip/formats/rar/rar5/decompressor"
         # RAR5 compression
         module Compression
           autoload :Store, "omnizip/formats/rar/rar5/compression/store"

data/lib/omnizip/formats/seven_zip/parser.rb

@@ -121,6 +121,19 @@ module Omnizip
         #
         # @param num_items [Integer] Number of items in bit vector
         # @return [Array<Boolean>] Bit vector
+        # Raw MSB-first bit vector without an all-defined marker
+        # (kEmptyStream/kEmptyFile/kAnti layout)
+        def read_raw_bit_vector(num_items)
+          num_bytes = (num_items + 7) / 8
+          raise EOFError if @position + num_bytes > @data.bytesize
+
+          bits_data = @data[@position, num_bytes]
+          @position += num_bytes
+          decode_bit_vector(bits_data, num_items)
+        end
+
+        # Bit vector with an all-defined marker byte (kCRC/kWinAttrib
+        # defined-bits layout)
         def read_bit_vector(num_items)
           all_defined = read_byte
           num_bytes = (num_items + 7) / 8
@@ -519,11 +532,11 @@ module Omnizip
         # @param entries [Array<Models::FileEntry>] File entries
         def read_empty_stream(entries)
           skip_size
-          empty_stream = read_bit_vector(entries.size)
+          # kEmptyStream carries a RAW bit vector — no all-defined
+          # marker byte (unlike kCRC/kWinAttrib)
+          empty_stream = read_raw_bit_vector(entries.size)
           entries.each_with_index do |entry, i|
-            # Bit vector: 0 = has stream (file), 1 = empty (directory)
-            # Convert to boolean: has_stream = (bit == 0)
-            entry.has_stream = (empty_stream[i].zero?)
+            entry.has_stream = empty_stream[i].zero?
             entry.is_dir = (empty_stream[i] == 1)
           end
         end
@@ -534,9 +547,11 @@ module Omnizip
         def read_empty_file(entries)
           skip_size
           empty_files = entries.reject(&:has_stream)
-          empty_bits = read_bit_vector(empty_files.size)
+          empty_bits = read_raw_bit_vector(empty_files.size)
           empty_files.each_with_index do |entry, i|
-            entry.is_empty = !empty_bits[i]
+            # Bit set = zero-byte FILE (not a directory)
+            entry.is_empty = empty_bits[i] == 1
+            entry.is_dir = false if entry.is_empty
           end
         end
 
@@ -546,7 +561,7 @@ module Omnizip
         def read_anti(entries)
           skip_size
           anti_files = entries.select { |e| !e.has_stream && !e.is_empty }
-          anti_bits = read_bit_vector(anti_files.size)
+          anti_bits = read_raw_bit_vector(anti_files.size)
           anti_files.each_with_index do |entry, i|
             entry.is_anti = anti_bits[i]
           end