cabriolet 0.1.2 → 0.2.0

data/lib/cabriolet/decompressors/lzss.rb

@@ -40,13 +40,17 @@ module Cabriolet
 
       # Decompress LZSS data
       #
-      # @param bytes [Integer] Number of bytes to decompress (unused, reads
-      #   until EOF)
+      # @param bytes [Integer, nil] Maximum number of output bytes to write (nil or 0 = until EOF)
       # @return [Integer] Number of bytes decompressed
-      def decompress(_bytes)
+      def decompress(bytes = nil)
         bytes_written = 0
+        # Only enforce limit if bytes is a positive integer
+        enforce_limit = bytes&.positive?
 
         loop do
+          # Check if we've reached the output byte limit (only when limit is enforced)
+          break if enforce_limit && bytes_written >= bytes
+
           # Read control byte
           control_byte = read_input_byte
           break if control_byte.nil?
@@ -55,6 +59,9 @@ module Cabriolet
 
           # Process each bit in the control byte
           8.times do |bit_index|
+            # Check output limit before each operation (only when limit is enforced)
+            break if enforce_limit && bytes_written >= bytes
+
             mask = 1 << bit_index
 
             if control_byte.anybits?(mask)
@@ -81,6 +88,9 @@ module Cabriolet
 
               # Copy from window
               length.times do
+                # Check if we've reached the limit mid-match
+                break if enforce_limit && bytes_written >= bytes
+
                 byte = @window[match_pos]
                 @window[@window_pos] = byte
                 write_output_byte(byte)

data/lib/cabriolet/decompressors/lzx.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "base"
+
 module Cabriolet
   module Decompressors
     # LZX handles LZX compressed data
@@ -100,7 +102,7 @@ module Cabriolet
       # @param output_length [Integer] Expected output length for E8 processing
       # @param is_delta [Boolean] Whether this is LZX DELTA format
       def initialize(io_system, input, output, buffer_size, window_bits:,
-                     reset_interval: 0, output_length: 0, is_delta: false)
+                     reset_interval: 0, output_length: 0, is_delta: false, salvage: false, **_kwargs)
         super(io_system, input, output, buffer_size)
 
         # Validate window_bits
@@ -146,8 +148,9 @@ module Cabriolet
         @intel_started = false
         @e8_buf = "\0" * FRAME_SIZE
 
-        # Initialize bitstream
-        @bitstream = Binary::Bitstream.new(io_system, input, buffer_size)
+        # Initialize bitstream (LZX uses MSB-first bit ordering per libmspack lzxd.c)
+        @bitstream = Binary::Bitstream.new(io_system, input, buffer_size,
+                                           bit_order: :msb, salvage: salvage)
 
         # Initialize Huffman trees
         initialize_trees
@@ -173,19 +176,21 @@ module Cabriolet
       def decompress(bytes)
         return 0 if bytes <= 0
 
+        # Read Intel filesize header if not already read (once per stream)
+        read_intel_header unless @header_read
+
         total_written = 0
         end_frame = ((@offset + bytes) / FRAME_SIZE) + 1
 
         while @frame < end_frame
-          # Check reset interval
-          reset_state if @reset_interval.positive? && (@frame % @reset_interval).zero?
+          # Check reset interval - reset offset registers at frame boundaries
+          if @reset_interval.positive? && (@frame % @reset_interval).zero? && @frame.positive?
+            @r0 = @r1 = @r2 = 1
+          end
 
           # Read DELTA chunk size if needed
           @bitstream.read_bits(16) if @is_delta
 
-          # Read Intel filesize header if needed
-          read_intel_header unless @header_read
-
           # Calculate frame size
           frame_size = calculate_frame_size
 
@@ -238,6 +243,10 @@ module Cabriolet
 
       # Reset LZX state (called at reset intervals)
       #
+      # Per libmspack: Only reset state variables, NOT Huffman code lengths.
+      # Lengths persist across blocks and are updated via delta encoding.
+      # They are only zeroed at initialization (in initialize_trees).
+      #
       # @return [void]
       def reset_state
         @r0 = 1
@@ -247,12 +256,17 @@ module Cabriolet
         @block_remaining = 0
         @block_type = BLOCKTYPE_INVALID
 
-        # Reset tree lengths to 0
-        @maintree_lengths.fill(0)
-        @length_lengths.fill(0)
+        # NOTE: Do NOT reset @maintree_lengths or @length_lengths here!
+        # Per libmspack lzxd.c line 267-269, lengths are initialized to 0
+        # only once (at start) "because deltas will be applied to them".
+        # Resetting them here breaks delta encoding between blocks.
       end
 
-      # Read Intel filesize header
+      # Read Intel filesize header (once per stream, before any frames)
+      #
+      # Format per libmspack:
+      # - 1 bit: Intel flag (if 0, filesize = 0; if 1, read 32-bit filesize)
+      # - If flag is 1: 32 bits for filesize (16 bits high, 16 bits low)
       #
       # @return [void]
       def read_intel_header
@@ -304,13 +318,20 @@ module Cabriolet
 
       # Read block header
       #
+      # LZX block header format (per libmspack):
+      # - 3 bits: block_type
+      # - 24 bits: block_length (16 bits high, 8 bits low, combined as (high << 8) | low)
+      #
       # @return [void]
       def read_block_header
-        # Align for uncompressed blocks
+        # Align for uncompressed blocks - this ensures correct byte alignment
+        # when reading the R0, R1, R2 values from the block header
         @bitstream.byte_align if @block_type == BLOCKTYPE_UNCOMPRESSED && @block_length.allbits?(1)
 
-        # Read block type and length
+        # Read block type (3 bits)
         @block_type = @bitstream.read_bits(3)
+
+        # Read block length (24 bits: 16 bits high, then 8 bits low)
         high = @bitstream.read_bits(16)
         low = @bitstream.read_bits(8)
         @block_length = (high << 8) | low
@@ -324,6 +345,8 @@ module Cabriolet
         when BLOCKTYPE_UNCOMPRESSED
           read_uncompressed_block_header
         else
+          # Per libmspack lzxd.c line 519-521, BLOCKTYPE_INVALID (0) and
+          # blocktypes 4-7 are all invalid and should raise an error
           raise DecompressionError, "Invalid block type: #{@block_type}"
         end
       end
@@ -338,11 +361,11 @@ module Cabriolet
         end
 
         # Build aligned tree
-        @aligned_tree = Huffman::Tree.new(@aligned_lengths, ALIGNED_MAXSYMBOLS)
-        unless @aligned_tree.build_table(ALIGNED_TABLEBITS)
-          raise DecompressionError,
-                "Failed to build aligned tree"
-        end
+        # Note: Aligned tree may be incomplete (Kraft sum < 1.0), which is valid
+        # as long as the unused codes are never encountered in the bitstream
+        @aligned_tree = Huffman::Tree.new(@aligned_lengths, ALIGNED_MAXSYMBOLS,
+                                          bit_order: :msb)
+        @aligned_tree.build_table(ALIGNED_TABLEBITS)
 
         # Read main and length trees (same as verbatim)
         read_main_and_length_trees
@@ -359,15 +382,14 @@ module Cabriolet
       #
       # @return [void]
       def read_main_and_length_trees
-        # Read and build pretree
-        read_pretree
-
         # Read main tree lengths using pretree
+        # Note: Each call to read_lengths reads its own pretree (per libmspack lzxd_read_lens)
         read_lengths(@maintree_lengths, 0, 256)
         read_lengths(@maintree_lengths, 256, @maintree_maxsymbols)
 
         # Build main tree
-        @maintree = Huffman::Tree.new(@maintree_lengths, @maintree_maxsymbols)
+        @maintree = Huffman::Tree.new(@maintree_lengths, @maintree_maxsymbols,
+                                      bit_order: :msb)
         unless @maintree.build_table(LENGTH_TABLEBITS)
           raise DecompressionError,
                 "Failed to build main tree"
@@ -380,7 +402,8 @@ module Cabriolet
         read_lengths(@length_lengths, 0, NUM_SECONDARY_LENGTHS)
 
         # Build length tree (may be empty)
-        @length_tree = Huffman::Tree.new(@length_lengths, LENGTH_MAXSYMBOLS)
+        @length_tree = Huffman::Tree.new(@length_lengths, LENGTH_MAXSYMBOLS,
+                                         bit_order: :msb)
         if @length_tree.build_table(LENGTH_TABLEBITS)
           @length_empty = false
         else
@@ -401,7 +424,8 @@ module Cabriolet
           @pretree_lengths[i] = @bitstream.read_bits(4)
         end
 
-        @pretree = Huffman::Tree.new(@pretree_lengths, PRETREE_MAXSYMBOLS)
+        @pretree = Huffman::Tree.new(@pretree_lengths, PRETREE_MAXSYMBOLS,
+                                     bit_order: :msb)
         return if @pretree.build_table(PRETREE_TABLEBITS)
 
         raise DecompressionError, "Failed to build pretree"
@@ -409,11 +433,16 @@ module Cabriolet
 
       # Read code lengths using pretree
       #
+      # Per libmspack's lzxd_read_lens, each call reads its own pretree first
+      #
       # @param lengths [Array<Integer>] Target length array
       # @param first [Integer] First symbol index
       # @param last [Integer] Last symbol index (exclusive)
       # @return [void]
       def read_lengths(lengths, first, last)
+        # Read and build pretree (20 elements, 4 bits each)
+        read_pretree
+
         x = first
 
         while x < last
@@ -494,9 +523,9 @@ module Cabriolet
             @window_posn += 1
             run_length -= 1
           else
-            # Match: decode length and offset
-            decode_match(main_element, run_length)
-            run_length = 0 # Match decoding handles run_length internally
+            # Match: decode length and offset, then decrement run_length by match_length
+            match_length = decode_match(main_element, run_length)
+            run_length -= match_length
           end
         end
       end
@@ -504,8 +533,8 @@ module Cabriolet
       # Decode and copy a match
       #
       # @param main_element [Integer] Main tree symbol
-      # @param run_length [Integer] Remaining run length
-      # @return [void]
+      # @param run_length [Integer] Remaining run length (unused, kept for compatibility)
+      # @return [Integer] Match length (bytes consumed)
       def decode_match(main_element, _run_length)
         main_element -= NUM_CHARS
 
@@ -533,8 +562,10 @@ module Cabriolet
           match_offset = @r0
         when 1
           @r1, @r0 = @r0, @r1
+          match_offset = @r0
         when 2
           @r2, @r0 = @r0, @r2
+          match_offset = @r0
         else
           # Calculate offset from position slot
           extra = position_slot >= 36 ? 17 : EXTRA_BITS[position_slot]
@@ -573,6 +604,9 @@ module Cabriolet
 
         # Copy match
         copy_match(match_offset, match_length)
+
+        # Return match length so caller can decrement run_length
+        match_length
       end
 
       # Decode extended match length for LZX DELTA
@@ -608,9 +642,12 @@ module Cabriolet
       # @return [void]
       def copy_match(offset, length)
         if offset > @window_posn
-          # Match wraps around window
-          if offset > @offset && (offset - @window_posn).positive?
-            raise DecompressionError, "Match offset beyond stream"
+          # Match wraps around window - validate it doesn't read beyond available data
+          # Per libmspack lzxd.c lines 622-628: check if match offset goes beyond
+          # what has been decompressed so far (accounting for any reference data)
+          ref_data_size = 0 # We don't support reference data yet (LZX DELTA feature)
+          if offset > @offset && (offset - @window_posn) > ref_data_size
+            raise DecompressionError, "Match offset beyond LZX stream"
           end
 
           # Copy from end of window

data/lib/cabriolet/decompressors/mszip.rb

@@ -14,6 +14,13 @@ module Cabriolet
       DISTANCE_MAXSYMBOLS = 32
       DISTANCE_TABLEBITS = 6
 
+      # MSZIP signature bytes
+      SIGNATURE_BYTE_C = 0x43  # ASCII 'C'
+      SIGNATURE_BYTE_K = 0x4B  # ASCII 'K'
+
+      # Maximum bytes to search for CK signature (prevents infinite loops)
+      MAX_SIGNATURE_SEARCH = 10_000
+
       # Match lengths for literal codes 257-285
       LIT_LENGTHS = [
         3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27,
@@ -50,7 +57,8 @@ module Cabriolet
       # @param output [System::FileHandle, System::MemoryHandle] Output handle
       # @param buffer_size [Integer] Buffer size for I/O operations
       # @param fix_mszip [Boolean] Enable repair mode for corrupted data
-      def initialize(io_system, input, output, buffer_size, fix_mszip: false)
+      def initialize(io_system, input, output, buffer_size, fix_mszip: false,
+salvage: false, **_kwargs)
         super(io_system, input, output, buffer_size)
         @fix_mszip = fix_mszip
 
@@ -58,9 +66,11 @@ module Cabriolet
         @window = "\0" * FRAME_SIZE
         @window_posn = 0
         @bytes_output = 0
+        @window_offset = 0 # Offset into window for unconsumed data (for multi-file CFDATA blocks)
 
         # Initialize bitstream
-        @bitstream = Binary::Bitstream.new(io_system, input, buffer_size)
+        @bitstream = Binary::Bitstream.new(io_system, input, buffer_size,
+                                           salvage: salvage)
 
         # Initialize Huffman trees
         @literal_lengths = Array.new(LITERAL_MAXSYMBOLS, 0)
@@ -76,15 +86,50 @@ module Cabriolet
       def decompress(bytes)
         total_written = 0
 
+        if ENV["DEBUG_MSZIP"]
+          warn "DEBUG MSZIP.decompress(#{bytes}): ENTRY bytes_output=#{@bytes_output} window_offset=#{@window_offset} window_posn=#{@window_posn}"
+        end
+
         while bytes.positive?
-          # Read 'CK' signature
-          read_signature
+          # Check if we have buffered data from previous inflate
+          if @bytes_output.positive?
+            if ENV["DEBUG_MSZIP"]
+              warn "DEBUG MSZIP: Using buffered data: bytes_output=#{@bytes_output} window_offset=#{@window_offset}"
+            end
 
-          # Reset window state for new block
+            # Write from buffer
+            write_amount = [bytes, @bytes_output].min
+            io_system.write(output, @window[@window_offset, write_amount])
+            total_written += write_amount
+            bytes -= write_amount
+            @bytes_output -= write_amount
+            @window_offset += write_amount
+
+            if ENV["DEBUG_MSZIP"]
+              warn "DEBUG MSZIP: After buffer write: total_written=#{total_written} bytes_remaining=#{bytes} bytes_output=#{@bytes_output}"
+            end
+
+            # Continue loop to check if we need more data
+            next
+          end
+
+          # No buffered data - need to inflate a new MSZIP frame
+          # Reset window for new frame
+          @window_offset = 0
           @window_posn = 0
-          @bytes_output = 0
 
-          # Inflate the block
+          # Read 'CK' signature (marks start of MSZIP frame)
+          # Every MSZIP frame starts with a CK signature
+          if ENV["DEBUG_MSZIP"]
+            warn "DEBUG MSZIP: Reading CK signature (new MSZIP frame)"
+          end
+          read_signature
+
+          # Inflate the MSZIP frame (processes deflate blocks until last_block or window full)
+          if ENV["DEBUG_MSZIP"]
+            warn "DEBUG MSZIP: Calling inflate_block"
+          end
+
           begin
             inflate_block
           rescue DecompressionError
@@ -97,11 +142,15 @@ module Cabriolet
             @bytes_output = FRAME_SIZE
           end
 
-          # Write output
-          write_amount = [bytes, @bytes_output].min
-          io_system.write(output, @window[0, write_amount])
-          total_written += write_amount
-          bytes -= write_amount
+          if ENV["DEBUG_MSZIP"]
+            warn "DEBUG MSZIP: After inflate_block: bytes_output=#{@bytes_output} window_posn=#{@window_posn}"
+          end
+
+          # Now we have data in the window buffer - loop back to write from it
+        end
+
+        if ENV["DEBUG_MSZIP"]
+          warn "DEBUG MSZIP.decompress: EXIT total_written=#{total_written}"
         end
 
         total_written
@@ -111,49 +160,63 @@ module Cabriolet
 
       # Read and verify 'CK' signature
       def read_signature
+        if ENV["DEBUG_MSZIP"]
+          warn "DEBUG read_signature: Before byte_align"
+        end
+
         # Align to byte boundary
         @bitstream.byte_align
 
-        # Read bytes until we find 'CK'
-        state = 0
-        bytes_read = 0
-        max_search = 10_000 # Prevent infinite loops
+        # Read first 2 bytes
+        c = @bitstream.read_bits(8)
+        k = @bitstream.read_bits(8)
 
-        loop do
-          byte = @bitstream.read_bits(8)
-          bytes_read += 1
+        if ENV["DEBUG_MSZIP"]
+          warn "DEBUG read_signature: Read 0x#{c.to_s(16)} 0x#{k.to_s(16)} (expected 'C'=0x43 'K'=0x4B)"
+        end
 
-          # Check for EOF (bitstream returns 0)
-          if bytes_read > 2 && byte.zero?
-            raise DecompressionError,
-                  "Unexpected EOF while searching for CK signature"
+        # If not CK, search for it (similar to libmspack's tolerant behavior)
+        unless c == SIGNATURE_BYTE_C && k == SIGNATURE_BYTE_K
+          # Search for CK signature in the stream (up to a reasonable limit)
+          max_search = 256
+          found = false
+
+          max_search.times do
+            # Shift: c becomes k, read new k
+            c = k
+            k = @bitstream.read_bits(8)
+
+            if c == SIGNATURE_BYTE_C && k == SIGNATURE_BYTE_K
+              found = true
+              if ENV["DEBUG_MSZIP"]
+                warn "DEBUG read_signature: Found CK signature after searching"
+              end
+              break
+            end
           end
 
-          # Prevent infinite loops
-          if bytes_read > max_search
+          unless found
             raise DecompressionError,
-                  "CK signature not found in stream"
-          end
-
-          if byte == 0x43 # 'C'
-            state = 1
-          elsif state == 1 && byte == 0x4B # 'K'
-            break
-          else
-            state = 0
+                  "Invalid MSZIP signature: could not find CK in stream"
           end
         end
       end
 
       # Inflate a single block
+      #
+      # Processes deflate blocks until the last_block flag is set or window is full.
+      # Always decodes complete blocks - does not stop mid-block.
       def inflate_block
-        loop do
-          # Read last block flag
-          last_block = @bitstream.read_bits(1)
+        # Read first block header
+        last_block = @bitstream.read_bits(1)
+        block_type = @bitstream.read_bits(2)
 
-          # Read block type
-          block_type = @bitstream.read_bits(2)
+        if ENV["DEBUG_MSZIP"]
+          warn "DEBUG inflate_block: First block: last_block=#{last_block} block_type=#{block_type}"
+        end
 
+        loop do
+          # Process current block
           case block_type
           when 0
             inflate_stored_block
@@ -167,7 +230,16 @@ module Cabriolet
             raise DecompressionError, "Invalid block type: #{block_type}"
           end
 
+          if ENV["DEBUG_MSZIP"]
+            warn "DEBUG inflate_block: After block: last_block=#{last_block} window_posn=#{@window_posn}"
+          end
+
+          # Stop if this was the last block
           break if last_block == 1
+
+          # Read next block header (only if we need to continue)
+          last_block = @bitstream.read_bits(1)
+          block_type = @bitstream.read_bits(2)
         end
 
         # Flush remaining window data
@@ -306,13 +378,25 @@ module Cabriolet
       end
 
       # Inflate a Huffman-compressed block
+      #
+      # Always decodes until code 256 (END OF BLOCK)
       def inflate_huffman_block
+        symbol_count = 0
         loop do
+          if ENV["DEBUG_MSZIP_SYMBOLS"]
+            warn "DEBUG inflate_huffman_block: window_posn=#{@window_posn} bytes_output=#{@bytes_output}"
+          end
+
           # Decode symbol from literal tree
           code = Huffman::Decoder.decode_symbol(
             @bitstream, @literal_tree.table, LITERAL_TABLEBITS,
             @literal_lengths, LITERAL_MAXSYMBOLS
           )
+          symbol_count += 1
+
+          if ENV["DEBUG_MSZIP_SYMBOLS"] || ENV["DEBUG_MSZIP"]
+            warn "DEBUG inflate_huffman_block[#{symbol_count}]: decoded code=#{code} (#{'0x%02x' % code if code < 256})"
+          end
 
           if code < 256
             # Literal byte
@@ -321,6 +405,9 @@ module Cabriolet
             flush_window if @window_posn == FRAME_SIZE
           elsif code == 256
             # End of block
+            if ENV["DEBUG_MSZIP"] || ENV["DEBUG_MSZIP_SYMBOLS"]
+              warn "DEBUG inflate_huffman_block: END OF BLOCK (window_posn=#{@window_posn})"
+            end
             break
           else
             # Length/distance pair (LZ77 match)

data/lib/cabriolet/decompressors/quantum.rb

@@ -12,7 +12,7 @@ module Cabriolet
       FRAME_SIZE = 32_768
 
       # Match constants
-      MAX_MATCH = 1028
+      MAX_MATCH = 259
 
       # Position slot tables (same as in qtmd.c)
       POSITION_BASE = [
@@ -68,7 +68,8 @@ module Cabriolet
       # @param output [System::FileHandle, System::MemoryHandle] Output handle
       # @param buffer_size [Integer] Buffer size for I/O operations
       # @param window_bits [Integer] Window size parameter (10-21)
-      def initialize(io_system, input, output, buffer_size, window_bits: 10)
+      def initialize(io_system, input, output, buffer_size, window_bits: 10,
+**_kwargs)
         super(io_system, input, output, buffer_size)
 
         # Validate window_bits

data/lib/cabriolet/errors.rb

@@ -36,4 +36,7 @@ module Cabriolet
 
   # Raised when seek operation fails
   class SeekError < IOError; end
+
+  # Raised when plugin operations fail
+  class PluginError < Error; end
 end