herringbone 0.1.0 → 0.3.0

data/lib/herringbone/bloom_filter.rb

@@ -0,0 +1,370 @@
+# frozen_string_literal: true
+
+module Herringbone
+  # A Parquet Split Block Bloom Filter (parquet-format BloomFilter.md).
+  #
+  # The bitset is made of 32-byte blocks of eight 32-bit words. A value is hashed with XXH64 over
+  # the PLAIN encoding of its physical value (without the length prefix for BYTE_ARRAY); the upper
+  # 32 bits of the hash pick a block, and the lower 32 bits set one bit in each of its words.
+  #
+  # A filter answers "definitely not in the column chunk" or "maybe": +might_contain?+ never
+  # returns false for a value that was inserted. Values are given as Ruby values and converted
+  # like the writer converts them (the column's encoder), so a Date, Time, BigDecimal or UUID
+  # String hashes the same bytes as the stored value. Nulls are never in a bloom filter.
+  # The writer builds them (bloom_filters: option) and reads with where: consult them.
+  class BloomFilter
+    # The spec's eight salt constants, one per word of a block
+    SALT = [0x47b6137b, 0x44974d91, 0x8824ad5b, 0xa2b7289d, 0x705495c7, 0x2df1424b, 0x9efc4947, 0x5c6bfb31].freeze
+    S0, S1, S2, S3, S4, S5, S6, S7 = SALT
+    # Low 16 bits of the salts: (lo * salt) mod 2**32 is computed as
+    # (lo & 0xFFFF) * salt + ((lo >> 16) * (salt & 0xFFFF) << 16), which stays a Fixnum
+    L0, L1, L2, L3, L4, L5, L6, L7 = SALT.map { |s| s & 0xFFFF }
+    # Size of a block: eight 32-bit words
+    BLOCK_BYTES = 32
+    # Smallest bitset: a single block
+    MIN_BYTES = 32
+    # Largest bitset accepted when reading or building (128 MiB, the cap parquet-mr uses)
+    MAX_BYTES = 128 * 1024 * 1024
+    # Default cap for filters sized by the writer (as in parquet-mr)
+    DEFAULT_MAX_BYTES = 1024 * 1024
+    # Default false positive probability for filters sized by the writer
+    DEFAULT_FPP = 0.01
+    # 32-bit mask
+    M32 = 0xFFFF_FFFF
+    # 64-bit mask
+    M64 = 0xFFFF_FFFF_FFFF_FFFF
+    # Shorthand for the physical type constants
+    T = Format::Type
+    # Physical types a bloom filter can be built for (the spec does not define BOOLEAN hashing)
+    TYPES = [T::INT32, T::INT64, T::INT96, T::FLOAT, T::DOUBLE, T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY].freeze
+
+    # Bitset size in bytes for +ndv+ distinct values at false positive probability +fpp+, per the
+    # spec's formula (m = -8 * ndv / ln(1 - fpp ** (1/8)) bits), rounded up to a power of two
+    # and clamped to MIN_BYTES..max_bytes
+    #
+    # @param ndv [Integer] expected number of distinct values (values below 1 count as 1)
+    # @param fpp [Float] false positive probability, strictly between 0 and 1
+    # @param max_bytes [Integer] upper bound, clamped to MIN_BYTES..MAX_BYTES and rounded down to a
+    #   power of two
+    # @return [Integer] bitset size in bytes, a power of two
+    # @raise [ArgumentError] when +fpp+ is not between 0 and 1
+    def self.optimal_num_bytes(ndv, fpp = DEFAULT_FPP, max_bytes: DEFAULT_MAX_BYTES)
+      fpp = Float(fpp)
+      raise ArgumentError, "fpp must be between 0 and 1, got #{fpp}" unless fpp > 0 && fpp < 1
+      max_bytes = Integer(max_bytes).clamp(MIN_BYTES, MAX_BYTES)
+      max_bytes = 1 << (max_bytes.bit_length - 1) # a power of two
+      ndv = [Integer(ndv), 1].max
+      bits = -8.0 * ndv / Math.log(1 - fpp**(1.0 / 8))
+      bytes = (bits / 8).ceil
+      return max_bytes if bytes >= max_bytes
+      return MIN_BYTES if bytes <= MIN_BYTES
+      1 << (bytes - 1).bit_length
+    end
+
+    # XXH64 of the PLAIN encoding of a physical value of +type+ (as the column encoders produce it)
+    #
+    # @param value [Integer, Float, String, Array<Integer>] the physical value; for INT96 the
+    #   +[nanos_of_day, julian_day]+ pair the encoder produces
+    # @param type [Integer] Format::Type physical type
+    # @return [Integer] the unsigned 64-bit hash
+    # @raise [UnsupportedError] for BOOLEAN (or any type not in TYPES)
+    def self.hash_physical(value, type)
+      case type
+      when T::INT32 then XXHash.xxh64_u32(value)
+      when T::INT64 then XXHash.xxh64_u64(value)
+      when T::FLOAT then XXHash.xxh64_u32([value].pack("e").unpack1("L<"))
+      when T::DOUBLE then XXHash.xxh64_u64([value].pack("E").unpack1("Q<"))
+      when T::INT96 then XXHash.xxh64(value.pack("Q<L<"))
+      when T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY then XXHash.xxh64(value)
+      else raise UnsupportedError, "Bloom filters are not defined for #{T::NAMES.fetch(type, type)} values"
+      end
+    end
+
+    # Hashes of many physical values of +type+, converting them in bulk. With +distinct+, each
+    # distinct physical value is hashed once (floats are compared by their bytes, so -0.0 and 0.0,
+    # or NaNs with different payloads, stay apart as they hash differently).
+    #
+    # @param values [Array] physical values, as for .hash_physical
+    # @param type [Integer] Format::Type physical type
+    # @param distinct [Boolean] hash each distinct value once (the result is then shorter)
+    # @return [Array<Integer>] the unsigned 64-bit hashes
+    # @raise [UnsupportedError] for BOOLEAN (or any type not in TYPES)
+    def self.hash_physical_all(values, type, distinct: false)
+      case type
+      when T::INT32 then XXHash.xxh64_u32_all(distinct ? values.uniq : values)
+      when T::INT64 then XXHash.xxh64_u64_all(distinct ? values.uniq : values)
+      when T::FLOAT
+        words = values.pack("e*").unpack("L<*")
+        XXHash.xxh64_u32_all(distinct ? words.uniq! || words : words)
+      when T::DOUBLE
+        lanes = values.pack("E*").unpack("Q<*")
+        XXHash.xxh64_u64_all(distinct ? lanes.uniq! || lanes : lanes)
+      when T::INT96 then XXHash.xxh64_all((distinct ? values.uniq : values).map { |v| v.pack("Q<L<") })
+      when T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY then XXHash.xxh64_all(distinct ? values.uniq : values)
+      else raise UnsupportedError, "Bloom filters are not defined for #{T::NAMES.fetch(type, type)} values"
+      end
+    end
+
+    # Reads a filter (header and bitset) from +buf+ at +pos+. Returns nil for algorithms, hashes
+    # or compressions this implementation does not know.
+    #
+    # @param buf [String] bytes holding the BloomFilterHeader followed by the bitset
+    # @param pos [Integer] byte offset of the header in +buf+
+    # @param column [Schema::Column, nil] column the filter belongs to, for converting values
+    # @return [BloomFilter, nil] the filter, or nil when it is of an unsupported kind
+    # @raise [FormatError] when the bitset is shorter than the header announces
+    # @raise [Thrift::Error] when the header cannot be decoded
+    def self.decode(buf, pos = 0, column: nil)
+      reader = Thrift::Reader.new(buf, pos)
+      header = reader.read_struct(Format::BloomFilterHeader)
+      return nil unless supported_header?(header)
+      bitset = buf.byteslice(reader.pos, header.num_bytes)
+      raise FormatError, "Truncated bloom filter" if bitset.nil? || bitset.bytesize != header.num_bytes
+      new(bitset: bitset, column: column)
+    end
+
+    # Whether a header describes a filter this implementation can read: split block algorithm,
+    # XXH64, uncompressed, and a size that is a multiple of 32 bytes up to MAX_BYTES
+    #
+    # @param header [Format::BloomFilterHeader] decoded header
+    # @return [Boolean] true when supported
+    def self.supported_header?(header)
+      n = header.num_bytes
+      n.is_a?(Integer) && n >= BLOCK_BYTES && (n % BLOCK_BYTES).zero? && n <= MAX_BYTES &&
+        !(header.algorithm && header.algorithm.block).nil? &&
+        !(header.hash_function && header.hash_function.xxhash).nil? &&
+        (header.compression.nil? || !header.compression.uncompressed.nil?)
+    end
+
+    # @return [Schema::Column, nil] column whose encoder converts values, nil for raw Strings
+    attr_reader :column
+
+    # A filter of +num_bytes+ (a multiple of 32, normally a power of two), or one using an existing
+    # +bitset+ String. With a +column+ (a Schema::Column), values are Ruby values converted with
+    # the column's encoder; without one, values must be Strings and their bytes are hashed.
+    #
+    # @param num_bytes [Integer, nil] bitset size for an empty filter (MIN_BYTES when nil); ignored
+    #   with +bitset+
+    # @param bitset [String, nil] existing bitset (little-endian 32-bit words)
+    # @param column [Schema::Column, nil] column the filter is for
+    # @raise [ArgumentError] when the size (or +bitset+'s size) is not a multiple of 32 bytes or
+    #   exceeds MAX_BYTES
+    # @raise [UnsupportedError] when the column's physical type cannot have a bloom filter
+    def initialize(num_bytes = nil, bitset: nil, column: nil)
+      num_bytes = bitset ? bitset.bytesize : Integer(num_bytes || MIN_BYTES)
+      unless num_bytes >= BLOCK_BYTES && (num_bytes % BLOCK_BYTES).zero? && num_bytes <= MAX_BYTES
+        raise ArgumentError, "Bloom filter size must be a multiple of 32 bytes up to 128MB, got #{num_bytes}"
+      end
+      @words = bitset ? bitset.unpack("V*") : Array.new(num_bytes / 4, 0)
+      @num_blocks = @words.size / 8
+      @column = column
+      if column && !TYPES.include?(column.type)
+        raise UnsupportedError, "Bloom filters are not supported for #{T::NAMES[column.type]} column #{column.dotted_path}"
+      end
+    end
+
+    # @return [Integer] bitset size in bytes
+    def num_bytes = @words.size * 4
+
+    # Adds a value to the filter
+    #
+    # @param value [Object] Ruby value, converted with the column's encoder (a String without a
+    #   column)
+    # @return [BloomFilter] self
+    # @raise [ArgumentError] for nil or a value the column cannot encode
+    def insert(value)
+      insert_hash(hash_of(value))
+      self
+    end
+
+    # Whether the value may be in the filter. Never false for an inserted value; true for a
+    # value that was not inserted with about the false positive probability the filter was sized for.
+    #
+    # @param value [Object] Ruby value, converted with the column's encoder (a String without a
+    #   column)
+    # @return [Boolean] false when the value is definitely absent
+    # @raise [ArgumentError] for nil or a value the column cannot encode
+    def might_contain?(value)
+      might_contain_hash?(hash_of(value))
+    end
+
+    # XXH64 hash the filter uses for a Ruby +value+
+    #
+    # @param value [Object] Ruby value, converted with the column's encoder (a String without a
+    #   column)
+    # @return [Integer] the unsigned 64-bit hash
+    # @raise [ArgumentError] for nil, a value the column cannot encode, or (without a column) a
+    #   non-String
+    def hash_of(value)
+      raise ArgumentError, "Nulls are not recorded in bloom filters" if value.nil?
+      if @column
+        begin
+          physical = @column.encoder.call(value)
+        rescue ArgumentError, TypeError, NoMethodError, RangeError, EncodeError => e
+          raise ArgumentError, "#{value.inspect} is not a valid value for #{@column.dotted_path}: #{e.message}"
+        end
+        self.class.hash_physical(physical, @column.type)
+      else
+        raise ArgumentError, "Without a column, bloom filter values must be Strings, got #{value.class}" unless value.is_a?(String)
+        XXHash.xxh64(value)
+      end
+    end
+
+    # Sets the bits for a hash: the upper 32 bits pick the block, the lower 32 bits (multiplied
+    # by each salt) one bit in each of its eight words
+    #
+    # @param h [Integer] unsigned 64-bit XXH64 hash
+    # @return [BloomFilter] self
+    def insert_hash(h)
+      i = (((h >> 32) * @num_blocks) >> 32) << 3
+      x0 = h & 0xFFFF
+      x1 = (h >> 16) & 0xFFFF
+      w = @words
+      w[i] |= 1 << (((x0 * S0 + (x1 * L0 << 16)) & M32) >> 27)
+      w[i + 1] |= 1 << (((x0 * S1 + (x1 * L1 << 16)) & M32) >> 27)
+      w[i + 2] |= 1 << (((x0 * S2 + (x1 * L2 << 16)) & M32) >> 27)
+      w[i + 3] |= 1 << (((x0 * S3 + (x1 * L3 << 16)) & M32) >> 27)
+      w[i + 4] |= 1 << (((x0 * S4 + (x1 * L4 << 16)) & M32) >> 27)
+      w[i + 5] |= 1 << (((x0 * S5 + (x1 * L5 << 16)) & M32) >> 27)
+      w[i + 6] |= 1 << (((x0 * S6 + (x1 * L6 << 16)) & M32) >> 27)
+      w[i + 7] |= 1 << (((x0 * S7 + (x1 * L7 << 16)) & M32) >> 27)
+      self
+    end
+
+    # Single-bit masks, +BITS[i] == 1 << i+
+    BITS = Array.new(32) { |i| 1 << i }.freeze
+
+    # Inserts many hashes at once: faster than insert_hash, as the hashes (mostly Bignums) are
+    # split into 32-bit halves in bulk and the rest is Fixnum arithmetic
+    #
+    # @param hashes [Array<Integer>] unsigned 64-bit XXH64 hashes
+    # @return [BloomFilter] self
+    def insert_hashes(hashes)
+      halves = hashes.pack("Q<*").unpack("V*")
+      w = @words
+      num_blocks = @num_blocks
+      bit = BITS
+      j = 0
+      n = halves.size
+      while j < n
+        lo = halves[j]
+        i = halves[j + 1] * num_blocks / 4_294_967_296 * 8
+        x0 = lo & 0xFFFF
+        x1 = lo / 65_536
+        w[i] |= bit[(x0 * S0 + x1 * L0 * 65536) / 134_217_728 & 31]
+        w[i + 1] |= bit[(x0 * S1 + x1 * L1 * 65536) / 134_217_728 & 31]
+        w[i + 2] |= bit[(x0 * S2 + x1 * L2 * 65536) / 134_217_728 & 31]
+        w[i + 3] |= bit[(x0 * S3 + x1 * L3 * 65536) / 134_217_728 & 31]
+        w[i + 4] |= bit[(x0 * S4 + x1 * L4 * 65536) / 134_217_728 & 31]
+        w[i + 5] |= bit[(x0 * S5 + x1 * L5 * 65536) / 134_217_728 & 31]
+        w[i + 6] |= bit[(x0 * S6 + x1 * L6 * 65536) / 134_217_728 & 31]
+        w[i + 7] |= bit[(x0 * S7 + x1 * L7 * 65536) / 134_217_728 & 31]
+        j += 2
+      end
+      self
+    end
+
+    # Whether all the bits for a hash are set (see #insert_hash)
+    #
+    # @param h [Integer] unsigned 64-bit XXH64 hash
+    # @return [Boolean] false when the hashed value is definitely absent
+    def might_contain_hash?(h)
+      i = (((h >> 32) * @num_blocks) >> 32) << 3
+      x0 = h & 0xFFFF
+      x1 = (h >> 16) & 0xFFFF
+      w = @words
+      w[i][(((x0 * S0 + (x1 * L0 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 1][(((x0 * S1 + (x1 * L1 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 2][(((x0 * S2 + (x1 * L2 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 3][(((x0 * S3 + (x1 * L3 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 4][(((x0 * S4 + (x1 * L4 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 5][(((x0 * S5 + (x1 * L5 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 6][(((x0 * S6 + (x1 * L6 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 7][(((x0 * S7 + (x1 * L7 << 16)) & M32) >> 27)] == 1
+    end
+
+    # The raw bitset (little-endian 32-bit words)
+    #
+    # @return [String] binary String of #num_bytes bytes
+    def bitset = @words.pack("V*")
+
+    # Header and bitset, as stored in a Parquet file
+    #
+    # @return [String] Thrift-encoded BloomFilterHeader followed by the bitset (binary)
+    def encode
+      header.encode << bitset
+    end
+
+    # @return [String] size and, when set, the column's dotted path
+    def inspect
+      "#<#{self.class.name} #{num_bytes} bytes#{" for #{@column.dotted_path}" if @column}>"
+    end
+
+    private
+
+    # Header for this filter: split block algorithm, XXH64 hash, uncompressed
+    #
+    # @return [Format::BloomFilterHeader] the header
+    def header
+      Format::BloomFilterHeader.new(
+        num_bytes: num_bytes,
+        algorithm: Format::BloomFilterAlgorithm.new(block: Format::SplitBlockAlgorithm.new),
+        hash_function: Format::BloomFilterHash.new(xxhash: Format::XxHash.new),
+        compression: Format::BloomFilterCompression.new(uncompressed: Format::BloomFilterUncompressed.new)
+      )
+    end
+  end
+
+  class Reader
+    # Internal (used by reads with where:): the bloom filter of a column chunk. +column+ is a
+    # dotted path ("a.b"), an Array path or a Schema::Column. Returns a BloomFilter, or nil when
+    # the chunk has none (or one of an unknown kind).
+    #
+    # @param row_group_index [Integer] index of the row group
+    # @param column [String, Array<String, Symbol>, Symbol, Schema::Column] the leaf column
+    # @return [BloomFilter, nil] the filter, or nil when there is none or it is unsupported
+    # @raise [IndexError] when there is no such row group
+    # @raise [ArgumentError] when there is no such column
+    # @raise [FormatError] when the filter is truncated or its header cannot be decoded
+    def bloom_filter(row_group_index, column)
+      col = bloom_filter_column(column)
+      rg = row_groups.fetch(row_group_index) { raise IndexError, "No row group #{row_group_index}" }
+      meta = rg.columns.fetch(col.index).meta_data
+      offset = meta&.bloom_filter_offset
+      return nil unless offset
+      length = meta.bloom_filter_length
+      @io.seek(offset)
+      if length
+        buf = @io.read(length)
+        raise FormatError, "Truncated bloom filter" if buf.nil? || buf.bytesize < length
+        return BloomFilter.decode(buf, column: col)
+      end
+      # Without a length (older writers), read the header first, then the bitset it announces
+      head = @io.read(256) || "".b
+      reader = Thrift::Reader.new(head)
+      header = reader.read_struct(Format::BloomFilterHeader)
+      return nil unless BloomFilter.supported_header?(header)
+      @io.seek(offset + reader.pos)
+      bitset = @io.read(header.num_bytes)
+      raise FormatError, "Truncated bloom filter" if bitset.nil? || bitset.bytesize != header.num_bytes
+      BloomFilter.new(bitset: bitset, column: col)
+    rescue Thrift::Error => e
+      raise FormatError, "Corrupt bloom filter header for #{col.dotted_path}: #{e.message}"
+    end
+
+    private
+
+    # Resolves the +column+ argument of #bloom_filter to a leaf column
+    #
+    # @param column [String, Array<String, Symbol>, Symbol, Schema::Column] dotted path, path
+    #   Array or column
+    # @return [Schema::Column] the column
+    # @raise [ArgumentError] when the schema has no such column
+    def bloom_filter_column(column)
+      return column if column.is_a?(Schema::Column)
+      col = schema.column(column.is_a?(Array) ? column.map(&:to_s) : column.to_s)
+      raise ArgumentError, "No column #{column.inspect}" unless col
+      col
+    end
+  end
+end

data/lib/herringbone/byte_values.rb

@@ -17,8 +17,11 @@ module Herringbone
     # After this many values, give up on the dictionary if more than half of them are distinct
     CARDINALITY_CHECK_AT = 4096
 
+    # Whether String#append_as_bytes (Ruby 3.4+) is available for appending without re-encoding
     APPEND_AS_BYTES = "".respond_to?(:append_as_bytes)
 
+    # @param width [Integer, nil] byte length of each value for FIXED_LEN_BYTE_ARRAY, nil for BYTE_ARRAY
+    # @param dictionary [Boolean] start in dictionary mode; false appends raw bytes from the start
     def initialize(width: nil, dictionary: true)
       @width = width
       @bytes = String.new(encoding: Encoding::BINARY)
@@ -31,14 +34,21 @@ module Herringbone
       end
     end
 
+    # @return [Integer] number of values held
     def size
       @indices ? @indices.size : @count
     end
 
+    # @return [Boolean] true when no values are held
     def empty? = size.zero?
 
+    # @return [Boolean] true while still dictionary-encoding (not yet switched to raw bytes)
     def dictionary? = !@indices.nil?
 
+    # Appends a value, switching to raw bytes when the dictionary limits are exceeded.
+    # @param value [String] bytes of the value; for FIXED_LEN_BYTE_ARRAY it must be +width+ bytes
+    #   long (not checked here)
+    # @return [ByteValues] self
     def <<(value)
       if @indices
         index = @dictionary[value]
@@ -61,16 +71,20 @@ module Herringbone
     end
 
     # Removes the last value
+    # @return [void]
     def pop
       truncate(size - 1) unless size.zero?
     end
 
     # Supports the `slice!(n..)` form used to roll back a failed row
+    # @param range [Range] endless range; values from +range.begin+ on are dropped
+    # @return [void]
     def slice!(range)
       truncate(range.begin)
     end
 
     # Approximate memory held, used to size row groups
+    # @return [Integer] estimated bytes
     def memory_bytes
       if @indices
         # Each distinct value is a String object plus a Hash entry
@@ -82,6 +96,11 @@ module Herringbone
 
     # Returns [:dictionary, values, indices] when the column is worth dictionary-encoding,
     # otherwise [:plain, values]
+    #
+    # Dictionary encoding is kept unless there are more than 16 values and more than about half
+    # of them are distinct.
+    # @return [Array(Symbol, Array<String>, Array<Integer>), Array(Symbol, Array<String>)]
+    #   distinct values and an index per value, or all values in order
     def materialize
       if @indices
         keys = @dictionary.keys
@@ -93,6 +112,9 @@ module Herringbone
 
     private
 
+    # Appends +value+ in raw-bytes mode, recording its length for BYTE_ARRAY.
+    # @param value [String] bytes of the value, in any encoding
+    # @return [ByteValues] self
     def append_bytes(value)
       if value.encoding == Encoding::BINARY || value.ascii_only?
         @bytes << value
@@ -106,6 +128,8 @@ module Herringbone
       self
     end
 
+    # Leaves dictionary mode, re-appending every value held so far as raw bytes.
+    # @return [void]
     def switch_to_bytes
       keys = @dictionary.keys
       indices = @indices
@@ -113,6 +137,9 @@ module Herringbone
       indices.each { |i| append_bytes(keys[i]) }
     end
 
+    # Drops all values after the first +n+.
+    # @param n [Integer] number of values to keep
+    # @return [void]
     def truncate(n)
       if @indices
         @indices.slice!(n..)
@@ -128,6 +155,8 @@ module Herringbone
       end
     end
 
+    # Splits the raw bytes back into one binary String per value.
+    # @return [Array<String>]
     def strings
       if @lengths
         pos = 0

data/lib/herringbone/codecs/lz4.rb

@@ -5,17 +5,26 @@ module Herringbone
     # Pure-Ruby LZ4: raw block format (Parquet LZ4_RAW), Hadoop-framed blocks
     # (Parquet's deprecated LZ4) and a decoder for the LZ4 frame format.
     module LZ4
+      # Raised for corrupt, truncated or unsupported LZ4 input.
       class Error < StandardError; end
 
+      # Shortest match a sequence can encode; the token's match length is stored minus this.
       MIN_MATCH = 4
       LAST_LITERALS = 5 # the last 5 bytes of a block are always literals
       MFLIMIT = 12      # the last match must start at least 12 bytes before the end
+      # Largest backward distance a 2-byte match offset can express.
       MAX_OFFSET = 65_535
+      # Size of the compressor's hash table, in bits (16384 entries).
       HASH_LOG = 14
+      # Right shift that keeps the top HASH_LOG bits of the 32-bit multiplicative hash.
       HASH_SHIFT = 32 - HASH_LOG
+      # Controls how fast the compressor skips ahead through input without matches (as in the reference).
       SKIP_STRENGTH = 6
+      # Magic number at the start of an LZ4 frame (little-endian).
       FRAME_MAGIC = 0x184D2204
+      # Hadoop block header: big-endian 32-bit uncompressed size and compressed size.
       HADOOP_PREFIX = 8
+      # Shorthand for Encoding::BINARY.
       BINARY = Encoding::BINARY
       # String#unpack1 accepts offset: since Ruby 3.1
       UNPACK_OFFSET = begin
@@ -28,6 +37,10 @@ module Herringbone
       module_function
 
       # Decompress a raw LZ4 block that must expand to exactly uncompressed_size bytes.
+      # @param input [String] raw LZ4 block
+      # @param uncompressed_size [Integer] exact decompressed size, from the page header
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the block is corrupt or does not decode to +uncompressed_size+ bytes
       def decompress_block(input, uncompressed_size)
         src = binary(input)
         out = String.new(capacity: uncompressed_size, encoding: BINARY)
@@ -40,6 +53,10 @@ module Herringbone
 
       # Parquet LZ4 (codec 5). Tries Hadoop framing, then the LZ4 frame format,
       # then a bare raw block (Arrow falls back hadoop -> raw; some writers emitted frames).
+      # @param input [String] compressed page data
+      # @param uncompressed_size [Integer] exact decompressed size, from the page header
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if none of the three layouts decodes
       def decompress_hadoop(input, uncompressed_size)
         src = binary(input)
         result = try_hadoop(src, uncompressed_size)
@@ -58,6 +75,10 @@ module Herringbone
 
       # Decode LZ4 frame format data (one or more frames, skippable frames ignored).
       # Checksums are skipped, not verified.
+      # @param input [String] one or more concatenated LZ4 frames
+      # @param max_size [Integer] upper bound on the decompressed size
+      # @return [String] decompressed bytes in ASCII-8BIT, possibly shorter than +max_size+
+      # @raise [Error] on bad magic, truncation, unsupported frame features or output over +max_size+
       def decompress_frame(input, max_size)
         src = binary(input)
         n = src.bytesize
@@ -79,6 +100,8 @@ module Herringbone
       end
 
       # Compress into a single raw LZ4 block.
+      # @param input [String] bytes to compress
+      # @return [String] raw LZ4 block in ASCII-8BIT
       def compress_block(input)
         src = binary(input)
         n = src.bytesize
@@ -121,7 +144,7 @@ module Herringbone
             # Emit the sequence: token, literal run, offset, extended match length
             lit_len = ip - anchor
             ml = len - MIN_MATCH
-            out << (((lit_len < 15 ? lit_len : 15) << 4) | (ml < 15 ? ml : 15))
+            out << ((((lit_len < 15) ? lit_len : 15) << 4) | ((ml < 15) ? ml : 15))
             write_length(out, lit_len - 15) if lit_len >= 15
             out << src.byteslice(anchor, lit_len) if lit_len > 0
             offset = ip - ref
@@ -144,6 +167,8 @@ module Herringbone
       end
 
       # Single Hadoop-framed block: [BE uncompressed size][BE compressed size][raw block]
+      # @param input [String] bytes to compress
+      # @return [String] Hadoop-framed LZ4 data in ASCII-8BIT
       def compress_hadoop(input)
         src = binary(input)
         block = compress_block(src)
@@ -152,18 +177,31 @@ module Herringbone
 
       # -- internals --
 
+      # @param str [String] input in any encoding
+      # @return [String] +str+ itself if already binary, otherwise a binary copy
       def binary(str)
-        str.encoding == BINARY ? str : str.b
+        (str.encoding == BINARY) ? str : str.b
       end
 
+      # @param src [String] binary input
+      # @param i [Integer] byte offset
+      # @return [Integer] unsigned little-endian 32-bit value at +i+
       def le32(src, i)
         src.byteslice(i, 4).unpack1("V")
       end
 
+      # Same as le32 but via getbyte, for Rubies without unpack1(offset:).
+      # @param src [String] binary input
+      # @param i [Integer] byte offset
+      # @return [Integer] unsigned little-endian 32-bit value at +i+
       def u32(src, i)
         src.getbyte(i) | (src.getbyte(i + 1) << 8) | (src.getbyte(i + 2) << 16) | (src.getbyte(i + 3) << 24)
       end
 
+      # Appends the extension of a literal or match length: 255-bytes followed by the remainder.
+      # @param out [String] binary output buffer, appended to
+      # @param len [Integer] length minus the 15 already stored in the token
+      # @return [String] +out+
       def write_length(out, len)
         if len >= 255
           out << ("\xFF".b * (len / 255))
@@ -172,14 +210,27 @@ module Herringbone
         out << len
       end
 
+      # Appends the final, literals-only sequence that terminates every block.
+      # @param out [String] binary output buffer, appended to
+      # @param src [String] binary input
+      # @param anchor [Integer] offset of the first pending literal in +src+
+      # @param lit_len [Integer] number of trailing literal bytes (may be zero)
+      # @return [String, nil] +out+, or nil when there are no literals
       def emit_last_literals(out, src, anchor, lit_len)
-        out << ((lit_len < 15 ? lit_len : 15) << 4)
+        out << (((lit_len < 15) ? lit_len : 15) << 4)
         write_length(out, lit_len - 15) if lit_len >= 15
         out << src.byteslice(anchor, lit_len) if lit_len > 0
       end
 
       # Decode one raw block from src[ip...iend], appending to out (which may already
       # hold earlier data that matches can reference). out may not grow beyond limit.
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the block in +src+
+      # @param iend [Integer] offset just past the block
+      # @param out [String] binary output buffer, appended to
+      # @param limit [Integer] maximum total byte size of +out+
+      # @return [Integer] input offset where decoding stopped
+      # @raise [Error] if the block is truncated, has an invalid offset or overflows +limit+
       def decode_block(src, ip, iend, out, limit)
         while ip < iend
           token = src.getbyte(ip)
@@ -237,6 +288,9 @@ module Herringbone
       end
 
       # Arrow-compatible Hadoop frame parsing; returns nil if the data does not fit the framing.
+      # @param src [String] binary input
+      # @param uncompressed_size [Integer] exact decompressed size expected over all blocks
+      # @return [String, nil] decompressed bytes, or nil if +src+ is not valid Hadoop-framed LZ4
       def try_hadoop(src, uncompressed_size)
         n = src.bytesize
         return nil if n < HADOOP_PREFIX
@@ -262,6 +316,15 @@ module Herringbone
         out
       end
 
+      # Decodes the descriptor and data blocks of one LZ4 frame (after its magic number),
+      # skipping content size and checksums.
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the frame descriptor (just past the magic)
+      # @param n [Integer] byte size of +src+
+      # @param out [String] binary output buffer, appended to
+      # @param limit [Integer] maximum total byte size of +out+
+      # @return [Integer] offset just past the frame
+      # @raise [Error] on truncation, an unsupported version or dictionary, or output over +limit+
       def decode_frame(src, ip, n, out, limit)
         raise Error, "Truncated LZ4 frame descriptor" if ip + 3 > n
         flg = src.getbyte(ip)