cabriolet 0.1.2 → 0.2.1

data/lib/cabriolet/compressors/quantum.rb

@@ -1,75 +1,24 @@
 # frozen_string_literal: true
 
+require_relative "../quantum_shared"
+
 module Cabriolet
   module Compressors
     # Quantum compresses data using arithmetic coding and LZ77-based matching
     # Based on the Quantum decompressor and libmspack qtmd.c implementation
     #
-    # STATUS: Functional with known limitations
-    # - Literals: WORKING ✓
-    # - Short matches (3-13 bytes): WORKING ✓
-    # - Longer matches (14+ bytes): Limited support (known issue)
-    # - Simple data round-trips successfully
-    # - Complex repeated patterns may have issues
-    #
     # The Quantum method was created by David Stafford, adapted by Microsoft
     # Corporation.
+    #
+    # NOTE: This compressor is a work-in-progress. The arithmetic coding
+    # implementation needs refinement to match the decoder exactly.
+    # For now, this implementation focuses on correct structure.
     # rubocop:disable Metrics/ClassLength
     class Quantum < Base
-      # Frame size (32KB per frame)
-      FRAME_SIZE = 32_768
-
-      # Match constants
-      MIN_MATCH = 3
-      MAX_MATCH = 1028
-
-      # Position slot tables (same as decompressor)
-      POSITION_BASE = [
-        0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384,
-        512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12_288, 16_384,
-        24_576, 32_768, 49_152, 65_536, 98_304, 131_072, 196_608, 262_144,
-        393_216, 524_288, 786_432, 1_048_576, 1_572_864
-      ].freeze
-
-      EXTRA_BITS = [
-        0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
-        9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16,
-        17, 17, 18, 18, 19, 19
-      ].freeze
-
-      LENGTH_BASE = [
-        0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 18, 22, 26,
-        30, 38, 46, 54, 62, 78, 94, 110, 126, 158, 190, 222, 254
-      ].freeze
-
-      LENGTH_EXTRA = [
-        0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2,
-        3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0
-      ].freeze
+      include QuantumShared
 
       attr_reader :window_bits, :window_size
 
-      # Represents a symbol in an arithmetic coding model
-      class ModelSymbol
-        attr_accessor :sym, :cumfreq
-
-        def initialize(sym, cumfreq)
-          @sym = sym
-          @cumfreq = cumfreq
-        end
-      end
-
-      # Represents an arithmetic coding model
-      class Model
-        attr_accessor :shiftsleft, :entries, :syms
-
-        def initialize(syms, entries)
-          @syms = syms
-          @entries = entries
-          @shiftsleft = 4
-        end
-      end
-
       # Initialize Quantum compressor
       #
       # @param io_system [System::IOSystem] I/O system for reading/writing
@@ -77,7 +26,8 @@ module Cabriolet
       # @param output [System::FileHandle, System::MemoryHandle] Output handle
       # @param buffer_size [Integer] Buffer size for I/O operations
       # @param window_bits [Integer] Window size parameter (10-21)
-      def initialize(io_system, input, output, buffer_size, window_bits: 10)
+      def initialize(io_system, input, output, buffer_size, window_bits: 10,
+**_kwargs)
         super(io_system, input, output, buffer_size)
 
         # Validate window_bits
@@ -179,7 +129,6 @@ module Cabriolet
 
       # Compress a single frame
       def compress_frame(data)
-        # No header needed - the first 16 bits of encoded data will be read as C
         pos = 0
 
         while pos < data.bytesize
@@ -198,27 +147,25 @@ module Cabriolet
           end
         end
 
-        # Finish arithmetic coding - output final range
-        # We need to output enough bits to disambiguate the final range
+        # Finish arithmetic coding
         finish_arithmetic_coding
       end
 
-      # Finish arithmetic coding by outputting the final state
+      # Finish arithmetic coding
       def finish_arithmetic_coding
-        # Output enough bits to ensure decoder can decode correctly
-        # We need to output a value that falls within [L, H)
-        # A common approach is to output L plus half the range
-        @underflow_bits += 1
-        bit = if @l.anybits?(0x4000)
-                1
-              else
-                0
-              end
-        @bitstream.write_bits_msb(bit, 1)
-        @underflow_bits.times do
-          @bitstream.write_bits_msb(bit ^ 1, 1)
+        # Output pending underflow bits
+        if @underflow_bits.positive?
+          bit = if @l.anybits?(0x4000)
+                  1
+                else
+                  0
+                end
+          @bitstream.write_bits_msb(bit, 1)
+          @underflow_bits.times do
+            @bitstream.write_bits_msb(bit ^ 1, 1)
+          end
+          @underflow_bits = 0
         end
-        @underflow_bits = 0
       end
 
       # Find best match in the sliding window
@@ -335,7 +282,6 @@ module Cabriolet
       end
 
       # Encode a symbol using arithmetic coding
-      # This is the inverse of GET_SYMBOL macro in qtmd.c
       def encode_symbol(model, sym)
         # Find symbol index in model
         i = 0
@@ -346,33 +292,29 @@ module Cabriolet
                 "Symbol #{sym} not found in model"
         end
 
-        # Calculate range (matching decoder line 93, 101-102)
-        range = (@h - @l) + 1
+        # Calculate range - use decoder's formula
+        range = ((@h - @l) & 0xFFFF) + 1
         symf = model.syms[0].cumfreq
 
-        # Update H and L (matching decoder lines 103-104)
-        # Decoder uses syms[i-1] and syms[i], so encoder at index j
-        # should use syms[j] and syms[j+1] to make decoder land at i=j+1
-        # But decoder returns syms[i-1].sym, so it will return syms[j].sym ✓
+        # Update H and L
         @h = @l + ((model.syms[i].cumfreq * range) / symf) - 1
         @l += ((model.syms[i + 1].cumfreq * range) / symf)
 
-        # Update model frequencies (matching decoder line 106)
+        # Update model frequencies
         j = i
         while j >= 0
           model.syms[j].cumfreq += 8
           j -= 1
         end
 
-        # Check if model needs updating (matching decoder line 107)
+        # Check if model needs updating
         update_model(model) if model.syms[0].cumfreq > 3800
 
-        # Normalize range (matching decoder lines 109-121)
+        # Normalize range
         normalize_range
       end
 
       # Normalize arithmetic coding range and output bits
-      # This implements the encoder equivalent of the decoder's normalization (lines 109-121)
       def normalize_range
         loop do
           if (@l & 0x8000) == (@h & 0x8000)
@@ -395,37 +337,36 @@ module Cabriolet
             @h |= 0x4000
 
             # Can't normalize further
-
           end
 
-          # Shift range (both for underflow and MSB match cases)
+          # Shift range
           @l = (@l << 1) & 0xFFFF
           @h = ((@h << 1) | 1) & 0xFFFF
         end
       end
 
-      # Update model statistics (matching qtmd_update_model exactly)
+      # Update model statistics
       def update_model(model)
         model.shiftsleft -= 1
 
         if model.shiftsleft.positive?
-          # Simple shift (matching decoder lines 129-135)
+          # Simple shift
           (model.entries - 1).downto(0) do |i|
             model.syms[i].cumfreq >>= 1
             model.syms[i].cumfreq = model.syms[i + 1].cumfreq + 1 if model.syms[i].cumfreq <= model.syms[i + 1].cumfreq
           end
         else
-          # Full rebuild (matching decoder lines 137-163)
+          # Full rebuild
           model.shiftsleft = 50
 
-          # Convert cumfreq to frequencies (lines 139-145)
+          # Convert cumfreq to frequencies
           (0...model.entries).each do |i|
             model.syms[i].cumfreq -= model.syms[i + 1].cumfreq
             model.syms[i].cumfreq += 1
             model.syms[i].cumfreq >>= 1
           end
 
-          # Sort by frequency (selection sort for stability, lines 150-158)
+          # Sort by frequency
           (0...(model.entries - 1)).each do |i|
             ((i + 1)...model.entries).each do |j|
               if model.syms[i].cumfreq < model.syms[j].cumfreq
@@ -434,7 +375,7 @@ module Cabriolet
             end
           end
 
-          # Convert back to cumulative frequencies (lines 161-163)
+          # Convert back to cumulative frequencies
           (model.entries - 1).downto(0) do |i|
             model.syms[i].cumfreq += model.syms[i + 1].cumfreq
           end

data/lib/cabriolet/decompressors/base.rb

@@ -12,7 +12,7 @@ module Cabriolet
       # @param input [System::FileHandle, System::MemoryHandle] Input handle
       # @param output [System::FileHandle, System::MemoryHandle] Output handle
       # @param buffer_size [Integer] Buffer size for I/O operations
-      def initialize(io_system, input, output, buffer_size)
+      def initialize(io_system, input, output, buffer_size, **_kwargs)
         @io_system = io_system
         @input = input
         @output = output

data/lib/cabriolet/decompressors/lzss.rb

@@ -40,13 +40,17 @@ module Cabriolet
 
       # Decompress LZSS data
       #
-      # @param bytes [Integer] Number of bytes to decompress (unused, reads
-      #   until EOF)
+      # @param bytes [Integer, nil] Maximum number of output bytes to write (nil or 0 = until EOF)
       # @return [Integer] Number of bytes decompressed
-      def decompress(_bytes)
+      def decompress(bytes = nil)
         bytes_written = 0
+        # Only enforce limit if bytes is a positive integer
+        enforce_limit = bytes&.positive?
 
         loop do
+          # Check if we've reached the output byte limit (only when limit is enforced)
+          break if enforce_limit && bytes_written >= bytes
+
           # Read control byte
           control_byte = read_input_byte
           break if control_byte.nil?
@@ -55,6 +59,9 @@ module Cabriolet
 
           # Process each bit in the control byte
           8.times do |bit_index|
+            # Check output limit before each operation (only when limit is enforced)
+            break if enforce_limit && bytes_written >= bytes
+
             mask = 1 << bit_index
 
             if control_byte.anybits?(mask)
@@ -81,6 +88,9 @@ module Cabriolet
 
               # Copy from window
               length.times do
+                # Check if we've reached the limit mid-match
+                break if enforce_limit && bytes_written >= bytes
+
                 byte = @window[match_pos]
                 @window[@window_pos] = byte
                 write_output_byte(byte)

data/lib/cabriolet/decompressors/lzx.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "base"
+
 module Cabriolet
   module Decompressors
     # LZX handles LZX compressed data
@@ -100,7 +102,7 @@ module Cabriolet
       # @param output_length [Integer] Expected output length for E8 processing
       # @param is_delta [Boolean] Whether this is LZX DELTA format
       def initialize(io_system, input, output, buffer_size, window_bits:,
-                     reset_interval: 0, output_length: 0, is_delta: false)
+                     reset_interval: 0, output_length: 0, is_delta: false, salvage: false, **_kwargs)
         super(io_system, input, output, buffer_size)
 
         # Validate window_bits
@@ -146,8 +148,9 @@ module Cabriolet
         @intel_started = false
         @e8_buf = "\0" * FRAME_SIZE
 
-        # Initialize bitstream
-        @bitstream = Binary::Bitstream.new(io_system, input, buffer_size)
+        # Initialize bitstream (LZX uses MSB-first bit ordering per libmspack lzxd.c)
+        @bitstream = Binary::Bitstream.new(io_system, input, buffer_size,
+                                           bit_order: :msb, salvage: salvage)
 
         # Initialize Huffman trees
         initialize_trees
@@ -173,19 +176,21 @@ module Cabriolet
       def decompress(bytes)
         return 0 if bytes <= 0
 
+        # Read Intel filesize header if not already read (once per stream)
+        read_intel_header unless @header_read
+
         total_written = 0
         end_frame = ((@offset + bytes) / FRAME_SIZE) + 1
 
         while @frame < end_frame
-          # Check reset interval
-          reset_state if @reset_interval.positive? && (@frame % @reset_interval).zero?
+          # Check reset interval - reset offset registers at frame boundaries
+          if @reset_interval.positive? && (@frame % @reset_interval).zero? && @frame.positive?
+            @r0 = @r1 = @r2 = 1
+          end
 
           # Read DELTA chunk size if needed
           @bitstream.read_bits(16) if @is_delta
 
-          # Read Intel filesize header if needed
-          read_intel_header unless @header_read
-
           # Calculate frame size
           frame_size = calculate_frame_size
 
@@ -238,6 +243,10 @@ module Cabriolet
 
       # Reset LZX state (called at reset intervals)
       #
+      # Per libmspack: Only reset state variables, NOT Huffman code lengths.
+      # Lengths persist across blocks and are updated via delta encoding.
+      # They are only zeroed at initialization (in initialize_trees).
+      #
       # @return [void]
       def reset_state
         @r0 = 1
@@ -247,12 +256,17 @@ module Cabriolet
         @block_remaining = 0
         @block_type = BLOCKTYPE_INVALID
 
-        # Reset tree lengths to 0
-        @maintree_lengths.fill(0)
-        @length_lengths.fill(0)
+        # NOTE: Do NOT reset @maintree_lengths or @length_lengths here!
+        # Per libmspack lzxd.c line 267-269, lengths are initialized to 0
+        # only once (at start) "because deltas will be applied to them".
+        # Resetting them here breaks delta encoding between blocks.
       end
 
-      # Read Intel filesize header
+      # Read Intel filesize header (once per stream, before any frames)
+      #
+      # Format per libmspack:
+      # - 1 bit: Intel flag (if 0, filesize = 0; if 1, read 32-bit filesize)
+      # - If flag is 1: 32 bits for filesize (16 bits high, 16 bits low)
       #
       # @return [void]
       def read_intel_header
@@ -304,13 +318,20 @@ module Cabriolet
 
       # Read block header
       #
+      # LZX block header format (per libmspack):
+      # - 3 bits: block_type
+      # - 24 bits: block_length (16 bits high, 8 bits low, combined as (high << 8) | low)
+      #
       # @return [void]
       def read_block_header
-        # Align for uncompressed blocks
+        # Align for uncompressed blocks - this ensures correct byte alignment
+        # when reading the R0, R1, R2 values from the block header
         @bitstream.byte_align if @block_type == BLOCKTYPE_UNCOMPRESSED && @block_length.allbits?(1)
 
-        # Read block type and length
+        # Read block type (3 bits)
         @block_type = @bitstream.read_bits(3)
+
+        # Read block length (24 bits: 16 bits high, then 8 bits low)
         high = @bitstream.read_bits(16)
         low = @bitstream.read_bits(8)
         @block_length = (high << 8) | low
@@ -324,6 +345,8 @@ module Cabriolet
         when BLOCKTYPE_UNCOMPRESSED
           read_uncompressed_block_header
         else
+          # Per libmspack lzxd.c line 519-521, BLOCKTYPE_INVALID (0) and
+          # blocktypes 4-7 are all invalid and should raise an error
           raise DecompressionError, "Invalid block type: #{@block_type}"
         end
       end
@@ -338,11 +361,11 @@ module Cabriolet
         end
 
         # Build aligned tree
-        @aligned_tree = Huffman::Tree.new(@aligned_lengths, ALIGNED_MAXSYMBOLS)
-        unless @aligned_tree.build_table(ALIGNED_TABLEBITS)
-          raise DecompressionError,
-                "Failed to build aligned tree"
-        end
+        # Note: Aligned tree may be incomplete (Kraft sum < 1.0), which is valid
+        # as long as the unused codes are never encountered in the bitstream
+        @aligned_tree = Huffman::Tree.new(@aligned_lengths, ALIGNED_MAXSYMBOLS,
+                                          bit_order: :msb)
+        @aligned_tree.build_table(ALIGNED_TABLEBITS)
 
         # Read main and length trees (same as verbatim)
         read_main_and_length_trees
@@ -359,15 +382,14 @@ module Cabriolet
       #
       # @return [void]
       def read_main_and_length_trees
-        # Read and build pretree
-        read_pretree
-
         # Read main tree lengths using pretree
+        # Note: Each call to read_lengths reads its own pretree (per libmspack lzxd_read_lens)
         read_lengths(@maintree_lengths, 0, 256)
         read_lengths(@maintree_lengths, 256, @maintree_maxsymbols)
 
         # Build main tree
-        @maintree = Huffman::Tree.new(@maintree_lengths, @maintree_maxsymbols)
+        @maintree = Huffman::Tree.new(@maintree_lengths, @maintree_maxsymbols,
+                                      bit_order: :msb)
         unless @maintree.build_table(LENGTH_TABLEBITS)
           raise DecompressionError,
                 "Failed to build main tree"
@@ -380,7 +402,8 @@ module Cabriolet
         read_lengths(@length_lengths, 0, NUM_SECONDARY_LENGTHS)
 
         # Build length tree (may be empty)
-        @length_tree = Huffman::Tree.new(@length_lengths, LENGTH_MAXSYMBOLS)
+        @length_tree = Huffman::Tree.new(@length_lengths, LENGTH_MAXSYMBOLS,
+                                         bit_order: :msb)
         if @length_tree.build_table(LENGTH_TABLEBITS)
           @length_empty = false
         else
@@ -401,7 +424,8 @@ module Cabriolet
           @pretree_lengths[i] = @bitstream.read_bits(4)
         end
 
-        @pretree = Huffman::Tree.new(@pretree_lengths, PRETREE_MAXSYMBOLS)
+        @pretree = Huffman::Tree.new(@pretree_lengths, PRETREE_MAXSYMBOLS,
+                                     bit_order: :msb)
         return if @pretree.build_table(PRETREE_TABLEBITS)
 
         raise DecompressionError, "Failed to build pretree"
@@ -409,11 +433,16 @@ module Cabriolet
 
       # Read code lengths using pretree
       #
+      # Per libmspack's lzxd_read_lens, each call reads its own pretree first
+      #
       # @param lengths [Array<Integer>] Target length array
       # @param first [Integer] First symbol index
       # @param last [Integer] Last symbol index (exclusive)
       # @return [void]
       def read_lengths(lengths, first, last)
+        # Read and build pretree (20 elements, 4 bits each)
+        read_pretree
+
         x = first
 
         while x < last
@@ -494,9 +523,9 @@ module Cabriolet
             @window_posn += 1
             run_length -= 1
           else
-            # Match: decode length and offset
-            decode_match(main_element, run_length)
-            run_length = 0 # Match decoding handles run_length internally
+            # Match: decode length and offset, then decrement run_length by match_length
+            match_length = decode_match(main_element, run_length)
+            run_length -= match_length
           end
         end
       end
@@ -504,8 +533,8 @@ module Cabriolet
       # Decode and copy a match
       #
       # @param main_element [Integer] Main tree symbol
-      # @param run_length [Integer] Remaining run length
-      # @return [void]
+      # @param run_length [Integer] Remaining run length (unused, kept for compatibility)
+      # @return [Integer] Match length (bytes consumed)
       def decode_match(main_element, _run_length)
         main_element -= NUM_CHARS
 
@@ -533,8 +562,10 @@ module Cabriolet
           match_offset = @r0
         when 1
           @r1, @r0 = @r0, @r1
+          match_offset = @r0
         when 2
           @r2, @r0 = @r0, @r2
+          match_offset = @r0
         else
           # Calculate offset from position slot
           extra = position_slot >= 36 ? 17 : EXTRA_BITS[position_slot]
@@ -573,6 +604,9 @@ module Cabriolet
 
         # Copy match
         copy_match(match_offset, match_length)
+
+        # Return match length so caller can decrement run_length
+        match_length
       end
 
       # Decode extended match length for LZX DELTA
@@ -608,9 +642,12 @@ module Cabriolet
       # @return [void]
       def copy_match(offset, length)
         if offset > @window_posn
-          # Match wraps around window
-          if offset > @offset && (offset - @window_posn).positive?
-            raise DecompressionError, "Match offset beyond stream"
+          # Match wraps around window - validate it doesn't read beyond available data
+          # Per libmspack lzxd.c lines 622-628: check if match offset goes beyond
+          # what has been decompressed so far (accounting for any reference data)
+          ref_data_size = 0 # We don't support reference data yet (LZX DELTA feature)
+          if offset > @offset && (offset - @window_posn) > ref_data_size
+            raise DecompressionError, "Match offset beyond LZX stream"
           end
 
           # Copy from end of window