cabriolet 0.1.2 → 0.2.1

data/lib/cabriolet/binary/bitstream_writer.rb

@@ -4,6 +4,10 @@ module Cabriolet
   module Binary
     # BitstreamWriter provides bit-level I/O operations for writing compressed data
     class BitstreamWriter
+      # Pre-computed byte constants for fast single-byte writes
+      # Avoids repeated array packing for each byte written
+      BYTE_CONSTANTS = Array.new(256) { |i| [i].pack("C") }.freeze
+
       attr_reader :io_system, :handle, :buffer_size
 
       # Initialize a new bitstream writer
@@ -11,15 +15,22 @@ module Cabriolet
       # @param io_system [System::IOSystem] I/O system for writing data
       # @param handle [System::FileHandle, System::MemoryHandle] Handle to write to
       # @param buffer_size [Integer] Size of the output buffer
-      # @param msb_first [Boolean] Whether to write bits MSB-first (for Quantum)
+      # @param bit_order [Symbol] Bit ordering - :lsb (default) or :msb
+      # @param msb_first [Boolean] Deprecated: use bit_order instead
       def initialize(io_system, handle,
-buffer_size = Cabriolet.default_buffer_size, msb_first: false)
+ buffer_size = Cabriolet.default_buffer_size, bit_order: :lsb, msb_first: false)
         @io_system = io_system
         @handle = handle
         @buffer_size = buffer_size
-        @msb_first = msb_first
+
+        # Support legacy msb_first parameter or new bit_order parameter
+        @bit_order = msb_first ? :msb : bit_order
+        @msb_first = (@bit_order == :msb)
+
         @bit_buffer = 0
         @bits_in_buffer = 0
+        @accumulated = 0
+        @bits_accumulated = 0
       end
 
       # Write specified number of bits to the stream
@@ -34,16 +45,38 @@ buffer_size = Cabriolet.default_buffer_size, msb_first: false)
                 "Can only write 1-32 bits at a time"
         end
 
-        # Add bits to buffer (LSB first, like DEFLATE)
-        @bit_buffer |= ((value & ((1 << num_bits) - 1)) << @bits_in_buffer)
-        @bits_in_buffer += num_bits
+        # Delegate to MSB method if in MSB mode
+        if @bit_order == :msb
+          write_bits_msb_internal(value, num_bits)
+          return
+        end
 
-        # Flush complete bytes
-        while @bits_in_buffer >= 8
-          byte = @bit_buffer & 0xFF
-          write_byte(byte)
-          @bit_buffer >>= 8
-          @bits_in_buffer -= 8
+        # LSB-first mode (default)
+        # Mask value to num_bits
+        value &= (1 << num_bits) - 1
+
+        # Accumulate bits
+        @accumulated |= (value << @bits_accumulated)
+        @bits_accumulated += num_bits
+
+        # Transfer accumulated bits to buffer in 8-bit chunks
+        while @bits_accumulated >= 8
+          # Take the lowest 8 bits from accumulated
+          byte = @accumulated & 0xFF
+          @accumulated >>= 8
+          @bits_accumulated -= 8
+
+          # Add to buffer
+          @bit_buffer |= (byte << @bits_in_buffer)
+          @bits_in_buffer += 8
+
+          # Flush complete bytes from buffer
+          while @bits_in_buffer >= 8
+            flush_byte = @bit_buffer & 0xFF
+            write_byte(flush_byte)
+            @bit_buffer >>= 8
+            @bits_in_buffer -= 8
+          end
         end
       end
 
@@ -51,20 +84,44 @@ buffer_size = Cabriolet.default_buffer_size, msb_first: false)
       #
       # @return [void]
       def byte_align
-        return if @bits_in_buffer.zero?
+        if @bit_order == :msb
+          # MSB mode: align to 16-bit boundary (like Bitstream reader)
+          return if @bits_in_buffer.zero?
+
+          padding = (16 - @bits_in_buffer) % 16
+        else
+          # LSB mode: align to 8-bit boundary
+          return if @bits_accumulated.zero?
 
-        # Pad with zeros to complete the current byte
-        padding_bits = 8 - (@bits_in_buffer % 8)
-        write_bits(0, padding_bits) if padding_bits < 8
+          padding = (8 - @bits_accumulated) % 8
+        end
+        if padding.positive?
+          write_bits(0, padding)
+        end
       end
 
       # Flush any remaining bits in the buffer
       #
       # @return [void]
       def flush
+        # For MSB mode, use the special MSB flush
+        if @bit_order == :msb
+          flush_msb_internal
+          return
+        end
+
+        # LSB mode flush
+        # First flush any accumulated bits
+        if @bits_accumulated.positive?
+          byte = @accumulated & 0xFF
+          write_byte(byte)
+          @accumulated = 0
+          @bits_accumulated = 0
+        end
+
+        # Then flush buffer
         return if @bits_in_buffer.zero?
 
-        # Write any remaining bits (padded with zeros)
         byte = @bit_buffer & 0xFF
         write_byte(byte)
         @bit_buffer = 0
@@ -76,10 +133,70 @@ buffer_size = Cabriolet.default_buffer_size, msb_first: false)
       # @param byte [Integer] Byte value to write
       # @return [void]
       def write_byte(byte)
-        data = [byte].pack("C")
+        # Use pre-encoded byte constant for better performance
+        data = BYTE_CONSTANTS[byte]
+        # DEBUG
+        if ENV["DEBUG_BITSTREAM"]
+          warn "DEBUG write_byte: pos=#{@bits_in_buffer} byte=#{byte} (#{byte.to_s(2).rjust(
+            8, '0'
+          )})"
+        end
         @io_system.write(@handle, data)
       end
 
+      # Write bits in MSB-first mode (internal implementation)
+      # Matches the behavior of Bitstream's MSB mode for reading
+      #
+      # @param value [Integer] Value to write
+      # @param num_bits [Integer] Number of bits to write
+      # @return [void]
+      def write_bits_msb_internal(value, num_bits)
+        # Mask value to num_bits
+        value &= (1 << num_bits) - 1
+
+        # Add bits to buffer (MSB first - inject at left side)
+        @bit_buffer = (@bit_buffer << num_bits) | value
+        @bits_in_buffer += num_bits
+
+        # Flush complete 16-bit words
+        # The most significant bits are at the left of the buffer
+        # We want to extract the highest 16 bits and keep the rest
+        while @bits_in_buffer >= 16
+          # Extract the highest 16 bits by shifting right by (bits_in_buffer - 16)
+          # This moves the top 16 bits to positions 0-15
+          @bits_in_buffer -= 16
+          shift = @bits_in_buffer
+          word = (@bit_buffer >> shift) & 0xFFFF
+          # Write little-endian (LSB byte first, then MSB byte) to match Bitstream reader
+          write_byte(word & 0xFF)
+          write_byte((word >> 8) & 0xFF)
+        end
+      end
+
+      # Flush MSB buffer (internal implementation)
+      # Write remaining bits padded to 16-bit boundary
+      #
+      # @return [void]
+      def flush_msb_internal
+        return if @bits_in_buffer.zero?
+
+        # Pad to 16-bit boundary
+        padding = (16 - @bits_in_buffer) % 16
+        @bit_buffer <<= padding if padding.positive?
+        @bits_in_buffer += padding
+
+        # Write final 16-bit word
+        if @bits_in_buffer == 16
+          word = @bit_buffer & 0xFFFF
+          # Write little-endian (LSB byte first, then MSB byte) to match Bitstream reader
+          write_byte(word & 0xFF)
+          write_byte((word >> 8) & 0xFF)
+        end
+
+        @bit_buffer = 0
+        @bits_in_buffer = 0
+      end
+
       # Write a raw byte directly (for signatures, etc.)
       # This ensures the bit buffer is flushed first
       #
@@ -105,9 +222,21 @@ buffer_size = Cabriolet.default_buffer_size, msb_first: false)
       # @param num_bits [Integer] Number of bits to write
       # @return [void]
       def write_bits_be(value, num_bits)
-        num_bits.times do |i|
-          bit = (value >> (num_bits - 1 - i)) & 1
-          write_bits(bit, 1)
+        # Write full bytes first for better performance
+        full_bytes = num_bits / 8
+        remaining_bits = num_bits % 8
+
+        # Write complete bytes MSB first
+        full_bytes.times do |i|
+          byte_shift = num_bits - 8 - (i * 8)
+          byte = (value >> byte_shift) & 0xFF
+          write_bits(byte, 8)
+        end
+
+        # Write remaining bits
+        if remaining_bits.positive?
+          remaining_value = value & ((1 << remaining_bits) - 1)
+          write_bits(remaining_value, remaining_bits)
         end
       end
 

data/lib/cabriolet/binary/chm_structures.rb

@@ -83,8 +83,8 @@ module Cabriolet
       uint32 :quickref_size
     end
 
-    # LZX Control Data
-    class LZXControlData < BinData::Record
+    # CHM LZX Control Data
+    class CHMLZXControlData < BinData::Record
       endian :little
 
       uint32 :len

data/lib/cabriolet/binary/hlp_structures.rb

@@ -4,62 +4,283 @@ require "bindata"
 
 module Cabriolet
   module Binary
-    # HLP (Windows Help) file format binary structures
+    # HLP (Windows Help / QuickHelp) file format binary structures
     #
-    # NOTE: This implementation is based on the knowledge that HLP files use
-    # LZSS compression with MODE_MSHELP, but cannot be fully validated due to
-    # lack of test fixtures and incomplete libmspack implementation.
+    # Based on the QuickHelp binary format specification from DosHelp project.
+    # HLP files store help databases with topics, compression, and hyperlinks.
+    #
+    # Format overview:
+    # - Signature (2 bytes): 0x4C 0x4E ("LN")
+    # - File Header (68 bytes)
+    # - Topic Index (variable)
+    # - Context Strings (variable)
+    # - Context Map (variable)
+    # - Keywords (optional, variable)
+    # - Huffman Tree (optional, variable)
+    # - Topic Texts (variable, compressed)
     module HLPStructures
-      # HLP file signature (common Windows Help magic)
-      # Note: Actual signature may vary; this is a placeholder
-      SIGNATURE = "?_\x03\x00".b.freeze
+      # QuickHelp file signature: 0x4C, 0x4E ("LN")
+      SIGNATURE = "\x4C\x4E".b.freeze unless defined?(SIGNATURE)
+
+      # File attributes flags
+      module Attributes
+        CASE_SENSITIVE = 0x01 unless defined?(CASE_SENSITIVE)
+        LOCKED = 0x02 unless defined?(LOCKED)
+      end
+
+      # Control bytes for keyword compression
+      module ControlBytes
+        # Dictionary entry with optional space (0x10-0x17)
+        DICT_ENTRY_MIN = 0x10 unless defined?(DICT_ENTRY_MIN)
+        DICT_ENTRY_MAX = 0x17 unless defined?(DICT_ENTRY_MAX)
+
+        # Run of spaces (0x18)
+        SPACE_RUN = 0x18 unless defined?(SPACE_RUN)
 
-      # HLP file header
+        # Run of bytes (0x19)
+        BYTE_RUN = 0x19 unless defined?(BYTE_RUN)
+
+        # Escape byte (0x1A)
+        ESCAPE = 0x1A unless defined?(ESCAPE)
+      end
+
+      # Text style flags for topic lines
+      module TextStyle
+        NONE = 0x00 unless defined?(NONE)
+        BOLD = 0x01 unless defined?(BOLD)
+        ITALIC = 0x02 unless defined?(ITALIC)
+        UNDERLINE = 0x04 unless defined?(UNDERLINE)
+      end
+
+      # QuickHelp file header (70 bytes total: 2 byte signature + 68 byte header)
       #
-      # Structure (placeholder based on typical compressed formats):
-      # - 4 bytes: signature/magic
-      # - 2 bytes: version
-      # - 4 bytes: file count
-      # - 4 bytes: directory offset
-      class Header < BinData::Record
+      # Structure:
+      # - 2 bytes: signature (0x4C 0x4E)
+      # - 2 bytes: version (always 2)
+      # - 2 bytes: attributes (bit flags)
+      # - 1 byte: control character (usually ':' or 0xFF)
+      # - 1 byte: padding
+      # - 2 bytes: topic count
+      # - 2 bytes: context count
+      # - 1 byte: display width
+      # - 1 byte: padding
+      # - 2 bytes: predefined context count
+      # - 14 bytes: database name (null-terminated, null-padded)
+      # - 4 bytes: reserved
+      # - 4 bytes: topic index offset
+      # - 4 bytes: context strings offset
+      # - 4 bytes: context map offset
+      # - 4 bytes: keywords offset (0 if not used)
+      # - 4 bytes: huffman tree offset (0 if not used)
+      # - 4 bytes: topic text offset
+      # - 4 bytes: reserved
+      # - 4 bytes: reserved
+      # - 4 bytes: database size
+      class FileHeader < BinData::Record
         endian :little
 
-        string :signature, length: 4
+        string :signature, length: 2
         uint16 :version
-        uint32 :file_count
-        uint32 :directory_offset
+        uint16 :attributes
+        uint8  :control_character
+        uint8  :padding1
+        uint16 :topic_count
+        uint16 :context_count
+        uint8  :display_width
+        uint8  :padding2
+        uint16 :predefined_ctx_count
+        string :database_name, length: 14
+        uint32 :reserved1
+        uint32 :topic_index_offset
+        uint32 :context_strings_offset
+        uint32 :context_map_offset
+        uint32 :keywords_offset
+        uint32 :huffman_tree_offset
+        uint32 :topic_text_offset
+        uint32 :reserved2
+        uint32 :reserved3
+        uint32 :database_size
+      end
+
+      # Topic index entry (4 bytes per topic)
+      #
+      # Array of (topic_count + 1) DWORDs that specify offsets of topic texts.
+      # The last entry indicates the end of the last topic.
+      class TopicOffset < BinData::Record
+        endian :little
+        uint32 :offset
       end
 
-      # HLP file entry in directory
+      # Context map entry (2 bytes per context)
+      #
+      # Maps context strings to topic indices.
+      class ContextMapEntry < BinData::Record
+        endian :little
+        uint16 :topic_index
+      end
+
+      # Huffman tree node (2 bytes per node)
+      #
+      # Leaf node: bit 15 set, bits 0-7 contain symbol
+      # Internal node: bit 15 clear, node_value/2 is left child index, i+1 is right child
+      class HuffmanNode < BinData::Record
+        endian :little
+        int16 :node_value
+
+        # Check if this is a leaf node
+        def leaf?
+          node_value.negative?
+        end
+
+        # Get symbol for leaf node
+        def symbol
+          return nil unless leaf?
+
+          node_value & 0xFF
+        end
+
+        # Get left child index for internal node
+        def left_child_index
+          return nil if leaf?
+
+          node_value / 2
+        end
+      end
+
+      # Topic compressed header (2 bytes)
+      #
+      # Appears at the start of each compressed topic text.
+      class TopicHeader < BinData::Record
+        endian :little
+        uint16 :decompressed_length
+      end
+
+      # Windows Help (WinHelp) 3.x file header (28 bytes)
       #
       # Structure:
-      # - 4 bytes: filename length
-      # - N bytes: filename (null-terminated)
-      # - 4 bytes: offset in archive
-      # - 4 bytes: uncompressed size
-      # - 4 bytes: compressed size
-      # - 1 byte: compression flag (0 = uncompressed, 1 = LZSS)
-      class FileEntry < BinData::Record
+      # - 2 bytes: Magic number (0x35F3)
+      # - 2 bytes: Unknown/version
+      # - 4 bytes: Directory offset
+      # - 4 bytes: Free list offset
+      # - 4 bytes: File size
+      # - 12 bytes: Reserved/padding
+      class WinHelp3Header < BinData::Record
         endian :little
 
-        uint32 :filename_length
-        string :filename, read_length: :filename_length
-        uint32 :offset
-        uint32 :uncompressed_size
-        uint32 :compressed_size
-        uint8  :compression_flag
+        uint16 :magic # 0x35F3
+        uint16 :unknown
+        uint32 :directory_offset
+        uint32 :free_list_offset
+        uint32 :file_size
+        string :reserved, length: 12
       end
 
-      # Topic header (for compressed help topics)
+      # Windows Help (WinHelp) 4.x file header (32 bytes)
       #
       # Structure:
-      # - 4 bytes: uncompressed size
-      # - 4 bytes: compressed size
-      class TopicHeader < BinData::Record
+      # - 4 bytes: Magic number (0x3F5F0000 or similar)
+      # - 4 bytes: Directory offset
+      # - 4 bytes: Free list offset
+      # - 4 bytes: File size
+      # - 16 bytes: Reserved/unknown
+      class WinHelp4Header < BinData::Record
+        endian :little
+
+        uint32 :magic # 0x3F5F0000 or similar
+        uint32 :directory_offset
+        uint32 :free_list_offset
+        uint32 :file_size
+        string :reserved, length: 16
+      end
+
+      # WinHelp internal file directory entry
+      #
+      # Variable size structure:
+      # - 4 bytes: File size
+      # - 2 bytes: Starting block number
+      # - Variable: File name (null-terminated, aligned)
+      class WinHelpDirectoryEntry < BinData::Record
+        endian :little
+
+        uint32 :file_size
+        uint16 :starting_block
+        stringz :filename
+      end
+
+      # WinHelp B+ tree header (from FILEHEADER of directory)
+      #
+      # Structure from helpdeco:
+      # - 2 bytes: Magic (0x293B)
+      # - 2 bytes: Flags (bit 0x0002 always 1, bit 0x0400 1 if directory)
+      # - 2 bytes: PageSize (0x0400=1k if directory, 0x0800=2k else)
+      # - 16 bytes: Structure (string describing structure of data)
+      # - 2 bytes: MustBeZero (0)
+      # - 2 bytes: PageSplits (number of page splits Btree has suffered)
+      # - 2 bytes: RootPage (page number of Btree root page)
+      # - 2 bytes: MustBeNegOne (0xFFFF)
+      # - 2 bytes: TotalPages (number of Btree pages)
+      # - 2 bytes: NLevels (number of levels of Btree)
+      # - 4 bytes: TotalBtreeEntries (number of entries in Btree)
+      #
+      # Total: 38 bytes (not 30!)
+      class WinHelpBTreeHeader < BinData::Record
+        endian :little
+
+        uint16 :magic # 0x293B
+        uint16 :flags
+        uint16 :page_size
+        string :structure, length: 16
+        int16  :must_be_zero
+        int16  :page_splits
+        int16  :root_page
+        int16  :must_be_neg_one
+        int16  :total_pages
+        int16  :n_levels
+        int32  :total_btree_entries
+        # Total: 2 + 2 + 2 + 16 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 4 = 38 bytes
+      end
+
+      # WinHelp B+ tree leaf node header
+      #
+      # Structure at beginning of every leaf-page:
+      # - 2 bytes: Unknown (no ID to identify leaf-page)
+      # - 2 bytes: NEntries (number of entries in this leaf-page)
+      # - 2 bytes: PreviousPage (page number of preceeding leaf-page or -1)
+      # - 2 bytes: NextPage (page number of next leaf-page or -1)
+      class WinHelpBTreeNodeHeader < BinData::Record
+        endian :little
+
+        uint16 :unknown
+        int16  :n_entries
+        int16  :previous_page
+        int16  :next_page
+      end
+
+      # WinHelp B+ tree index node header (for internal nodes)
+      #
+      # Structure at beginning of every index-page:
+      # - 2 bytes: Unknown (no ID to identify index-page)
+      # - 2 bytes: NEntries (number of entries in this index-page)
+      # - 2 bytes: PreviousPage (page number of previous page)
+      class WinHelpBTreeIndexHeader < BinData::Record
+        endian :little
+
+        uint16 :unknown
+        int16  :n_entries
+        int16  :previous_page
+      end
+
+      # WinHelp FILEHEADER structure at FileOffset of each internal file
+      #
+      # - 4 bytes: ReservedSpace (reserved space in help file incl. FILEHEADER)
+      # - 4 bytes: UsedSpace (used space in help file excl. FILEHEADER)
+      # - 1 byte: FileFlags (normally 4)
+      class WinHelpFileHeader < BinData::Record
         endian :little
 
-        uint32 :uncompressed_size
-        uint32 :compressed_size
+        int32  :reserved_space
+        int32  :used_space
+        uint8  :file_flags
       end
     end
   end