herringbone 0.2.0 → 0.4.0

data/lib/herringbone/bloom_filter.rb

@@ -13,19 +13,27 @@ module Herringbone
   # String hashes the same bytes as the stored value. Nulls are never in a bloom filter.
   # The writer builds them (bloom_filters: option) and reads with where: consult them.
   class BloomFilter
+    # The spec's eight salt constants, one per word of a block
     SALT = [0x47b6137b, 0x44974d91, 0x8824ad5b, 0xa2b7289d, 0x705495c7, 0x2df1424b, 0x9efc4947, 0x5c6bfb31].freeze
     S0, S1, S2, S3, S4, S5, S6, S7 = SALT
     # Low 16 bits of the salts: (lo * salt) mod 2**32 is computed as
     # (lo & 0xFFFF) * salt + ((lo >> 16) * (salt & 0xFFFF) << 16), which stays a Fixnum
     L0, L1, L2, L3, L4, L5, L6, L7 = SALT.map { |s| s & 0xFFFF }
+    # Size of a block: eight 32-bit words
     BLOCK_BYTES = 32
+    # Smallest bitset: a single block
     MIN_BYTES = 32
+    # Largest bitset accepted when reading or building (128 MiB, the cap parquet-mr uses)
     MAX_BYTES = 128 * 1024 * 1024
     # Default cap for filters sized by the writer (as in parquet-mr)
     DEFAULT_MAX_BYTES = 1024 * 1024
+    # Default false positive probability for filters sized by the writer
     DEFAULT_FPP = 0.01
+    # 32-bit mask
     M32 = 0xFFFF_FFFF
+    # 64-bit mask
     M64 = 0xFFFF_FFFF_FFFF_FFFF
+    # Shorthand for the physical type constants
     T = Format::Type
     # Physical types a bloom filter can be built for (the spec does not define BOOLEAN hashing)
     TYPES = [T::INT32, T::INT64, T::INT96, T::FLOAT, T::DOUBLE, T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY].freeze
@@ -33,10 +41,17 @@ module Herringbone
     # Bitset size in bytes for +ndv+ distinct values at false positive probability +fpp+, per the
     # spec's formula (m = -8 * ndv / ln(1 - fpp ** (1/8)) bits), rounded up to a power of two
     # and clamped to MIN_BYTES..max_bytes
+    #
+    # @param ndv [Integer] expected number of distinct values (values below 1 count as 1)
+    # @param fpp [Float] false positive probability, strictly between 0 and 1
+    # @param max_bytes [Integer] upper bound, clamped to MIN_BYTES..MAX_BYTES and rounded down to a
+    #   power of two
+    # @return [Integer] bitset size in bytes, a power of two
+    # @raise [ArgumentError] when +fpp+ is not between 0 and 1
     def self.optimal_num_bytes(ndv, fpp = DEFAULT_FPP, max_bytes: DEFAULT_MAX_BYTES)
       fpp = Float(fpp)
       raise ArgumentError, "fpp must be between 0 and 1, got #{fpp}" unless fpp > 0 && fpp < 1
-      max_bytes = [[Integer(max_bytes), MAX_BYTES].min, MIN_BYTES].max
+      max_bytes = Integer(max_bytes).clamp(MIN_BYTES, MAX_BYTES)
       max_bytes = 1 << (max_bytes.bit_length - 1) # a power of two
       ndv = [Integer(ndv), 1].max
       bits = -8.0 * ndv / Math.log(1 - fpp**(1.0 / 8))
@@ -47,6 +62,12 @@ module Herringbone
     end
 
     # XXH64 of the PLAIN encoding of a physical value of +type+ (as the column encoders produce it)
+    #
+    # @param value [Integer, Float, String, Array<Integer>] the physical value; for INT96 the
+    #   +[nanos_of_day, julian_day]+ pair the encoder produces
+    # @param type [Integer] Format::Type physical type
+    # @return [Integer] the unsigned 64-bit hash
+    # @raise [UnsupportedError] for BOOLEAN (or any type not in TYPES)
     def self.hash_physical(value, type)
       case type
       when T::INT32 then XXHash.xxh64_u32(value)
@@ -62,6 +83,12 @@ module Herringbone
     # Hashes of many physical values of +type+, converting them in bulk. With +distinct+, each
     # distinct physical value is hashed once (floats are compared by their bytes, so -0.0 and 0.0,
     # or NaNs with different payloads, stay apart as they hash differently).
+    #
+    # @param values [Array] physical values, as for .hash_physical
+    # @param type [Integer] Format::Type physical type
+    # @param distinct [Boolean] hash each distinct value once (the result is then shorter)
+    # @return [Array<Integer>] the unsigned 64-bit hashes
+    # @raise [UnsupportedError] for BOOLEAN (or any type not in TYPES)
     def self.hash_physical_all(values, type, distinct: false)
       case type
       when T::INT32 then XXHash.xxh64_u32_all(distinct ? values.uniq : values)
@@ -80,6 +107,13 @@ module Herringbone
 
     # Reads a filter (header and bitset) from +buf+ at +pos+. Returns nil for algorithms, hashes
     # or compressions this implementation does not know.
+    #
+    # @param buf [String] bytes holding the BloomFilterHeader followed by the bitset
+    # @param pos [Integer] byte offset of the header in +buf+
+    # @param column [Schema::Column, nil] column the filter belongs to, for converting values
+    # @return [BloomFilter, nil] the filter, or nil when it is of an unsupported kind
+    # @raise [FormatError] when the bitset is shorter than the header announces
+    # @raise [Thrift::Error] when the header cannot be decoded
     def self.decode(buf, pos = 0, column: nil)
       reader = Thrift::Reader.new(buf, pos)
       header = reader.read_struct(Format::BloomFilterHeader)
@@ -89,48 +123,78 @@ module Herringbone
       new(bitset: bitset, column: column)
     end
 
+    # Whether a header describes a filter this implementation can read: split block algorithm,
+    # XXH64, uncompressed, and a size that is a multiple of 32 bytes up to MAX_BYTES
+    #
+    # @param header [Format::BloomFilterHeader] decoded header
+    # @return [Boolean] true when supported
     def self.supported_header?(header)
       n = header.num_bytes
       n.is_a?(Integer) && n >= BLOCK_BYTES && (n % BLOCK_BYTES).zero? && n <= MAX_BYTES &&
-        header.algorithm&.block && header.hash_function&.xxhash &&
-        (header.compression.nil? || header.compression.uncompressed)
+        !(header.algorithm && header.algorithm.block).nil? &&
+        !(header.hash_function && header.hash_function.xxhash).nil? &&
+        (header.compression.nil? || !header.compression.uncompressed.nil?)
     end
 
+    # @return [Schema::Column, nil] column whose encoder converts values, nil for raw Strings
     attr_reader :column
 
     # A filter of +num_bytes+ (a multiple of 32, normally a power of two), or one using an existing
     # +bitset+ String. With a +column+ (a Schema::Column), values are Ruby values converted with
     # the column's encoder; without one, values must be Strings and their bytes are hashed.
+    #
+    # @param num_bytes [Integer, nil] bitset size for an empty filter (MIN_BYTES when nil); ignored
+    #   with +bitset+
+    # @param bitset [String, nil] existing bitset (little-endian 32-bit words)
+    # @param column [Schema::Column, nil] column the filter is for
+    # @raise [ArgumentError] when the size (or +bitset+'s size) is not a multiple of 32 bytes or
+    #   exceeds MAX_BYTES
+    # @raise [UnsupportedError] when the column's physical type cannot have a bloom filter
     def initialize(num_bytes = nil, bitset: nil, column: nil)
-      if bitset
-        @words = bitset.unpack("V*")
-      else
-        num_bytes = Integer(num_bytes || MIN_BYTES)
-        unless num_bytes >= BLOCK_BYTES && (num_bytes % BLOCK_BYTES).zero? && num_bytes <= MAX_BYTES
-          raise ArgumentError, "Bloom filter size must be a multiple of 32 bytes up to 128MB, got #{num_bytes}"
-        end
-        @words = Array.new(num_bytes / 4, 0)
+      num_bytes = bitset ? bitset.bytesize : Integer(num_bytes || MIN_BYTES)
+      unless num_bytes >= BLOCK_BYTES && (num_bytes % BLOCK_BYTES).zero? && num_bytes <= MAX_BYTES
+        raise ArgumentError, "Bloom filter size must be a multiple of 32 bytes up to 128MB, got #{num_bytes}"
       end
+      @words = bitset ? bitset.unpack("V*") : Array.new(num_bytes / 4, 0)
       @num_blocks = @words.size / 8
-      raise ArgumentError, "Bloom filter bitset must be a multiple of 32 bytes" if @num_blocks.zero? || @words.size % 8 != 0
       @column = column
       if column && !TYPES.include?(column.type)
         raise UnsupportedError, "Bloom filters are not supported for #{T::NAMES[column.type]} column #{column.dotted_path}"
       end
     end
 
+    # @return [Integer] bitset size in bytes
     def num_bytes = @words.size * 4
 
+    # Adds a value to the filter
+    #
+    # @param value [Object] Ruby value, converted with the column's encoder (a String without a
+    #   column)
+    # @return [BloomFilter] self
+    # @raise [ArgumentError] for nil or a value the column cannot encode
     def insert(value)
       insert_hash(hash_of(value))
       self
     end
 
+    # Whether the value may be in the filter. Never false for an inserted value; true for a
+    # value that was not inserted with about the false positive probability the filter was sized for.
+    #
+    # @param value [Object] Ruby value, converted with the column's encoder (a String without a
+    #   column)
+    # @return [Boolean] false when the value is definitely absent
+    # @raise [ArgumentError] for nil or a value the column cannot encode
     def might_contain?(value)
       might_contain_hash?(hash_of(value))
     end
 
     # XXH64 hash the filter uses for a Ruby +value+
+    #
+    # @param value [Object] Ruby value, converted with the column's encoder (a String without a
+    #   column)
+    # @return [Integer] the unsigned 64-bit hash
+    # @raise [ArgumentError] for nil, a value the column cannot encode, or (without a column) a
+    #   non-String
     def hash_of(value)
       raise ArgumentError, "Nulls are not recorded in bloom filters" if value.nil?
       if @column
@@ -146,6 +210,11 @@ module Herringbone
       end
     end
 
+    # Sets the bits for a hash: the upper 32 bits pick the block, the lower 32 bits (multiplied
+    # by each salt) one bit in each of its eight words
+    #
+    # @param h [Integer] unsigned 64-bit XXH64 hash
+    # @return [BloomFilter] self
     def insert_hash(h)
       i = (((h >> 32) * @num_blocks) >> 32) << 3
       x0 = h & 0xFFFF
@@ -162,10 +231,14 @@ module Herringbone
       self
     end
 
+    # Single-bit masks, +BITS[i] == 1 << i+
     BITS = Array.new(32) { |i| 1 << i }.freeze
 
     # Inserts many hashes at once: faster than insert_hash, as the hashes (mostly Bignums) are
     # split into 32-bit halves in bulk and the rest is Fixnum arithmetic
+    #
+    # @param hashes [Array<Integer>] unsigned 64-bit XXH64 hashes
+    # @return [BloomFilter] self
     def insert_hashes(hashes)
       halves = hashes.pack("Q<*").unpack("V*")
       w = @words
@@ -191,6 +264,10 @@ module Herringbone
       self
     end
 
+    # Whether all the bits for a hash are set (see #insert_hash)
+    #
+    # @param h [Integer] unsigned 64-bit XXH64 hash
+    # @return [Boolean] false when the hashed value is definitely absent
     def might_contain_hash?(h)
       i = (((h >> 32) * @num_blocks) >> 32) << 3
       x0 = h & 0xFFFF
@@ -207,19 +284,27 @@ module Herringbone
     end
 
     # The raw bitset (little-endian 32-bit words)
+    #
+    # @return [String] binary String of #num_bytes bytes
     def bitset = @words.pack("V*")
 
     # Header and bitset, as stored in a Parquet file
+    #
+    # @return [String] Thrift-encoded BloomFilterHeader followed by the bitset (binary)
     def encode
       header.encode << bitset
     end
 
+    # @return [String] size and, when set, the column's dotted path
     def inspect
       "#<#{self.class.name} #{num_bytes} bytes#{" for #{@column.dotted_path}" if @column}>"
     end
 
     private
 
+    # Header for this filter: split block algorithm, XXH64 hash, uncompressed
+    #
+    # @return [Format::BloomFilterHeader] the header
     def header
       Format::BloomFilterHeader.new(
         num_bytes: num_bytes,
@@ -234,6 +319,13 @@ module Herringbone
     # Internal (used by reads with where:): the bloom filter of a column chunk. +column+ is a
     # dotted path ("a.b"), an Array path or a Schema::Column. Returns a BloomFilter, or nil when
     # the chunk has none (or one of an unknown kind).
+    #
+    # @param row_group_index [Integer] index of the row group
+    # @param column [String, Array<String, Symbol>, Symbol, Schema::Column] the leaf column
+    # @return [BloomFilter, nil] the filter, or nil when there is none or it is unsupported
+    # @raise [IndexError] when there is no such row group
+    # @raise [ArgumentError] when there is no such column
+    # @raise [FormatError] when the filter is truncated or its header cannot be decoded
     def bloom_filter(row_group_index, column)
       col = bloom_filter_column(column)
       rg = row_groups.fetch(row_group_index) { raise IndexError, "No row group #{row_group_index}" }
@@ -256,10 +348,18 @@ module Herringbone
       bitset = @io.read(header.num_bytes)
       raise FormatError, "Truncated bloom filter" if bitset.nil? || bitset.bytesize != header.num_bytes
       BloomFilter.new(bitset: bitset, column: col)
+    rescue Thrift::Error => e
+      raise FormatError, "Corrupt bloom filter header for #{col.dotted_path}: #{e.message}"
     end
 
     private
 
+    # Resolves the +column+ argument of #bloom_filter to a leaf column
+    #
+    # @param column [String, Array<String, Symbol>, Symbol, Schema::Column] dotted path, path
+    #   Array or column
+    # @return [Schema::Column] the column
+    # @raise [ArgumentError] when the schema has no such column
     def bloom_filter_column(column)
       return column if column.is_a?(Schema::Column)
       col = schema.column(column.is_a?(Array) ? column.map(&:to_s) : column.to_s)

data/lib/herringbone/byte_values.rb

@@ -17,8 +17,11 @@ module Herringbone
     # After this many values, give up on the dictionary if more than half of them are distinct
     CARDINALITY_CHECK_AT = 4096
 
+    # Whether String#append_as_bytes (Ruby 3.4+) is available for appending without re-encoding
     APPEND_AS_BYTES = "".respond_to?(:append_as_bytes)
 
+    # @param width [Integer, nil] byte length of each value for FIXED_LEN_BYTE_ARRAY, nil for BYTE_ARRAY
+    # @param dictionary [Boolean] start in dictionary mode; false appends raw bytes from the start
     def initialize(width: nil, dictionary: true)
       @width = width
       @bytes = String.new(encoding: Encoding::BINARY)
@@ -31,14 +34,21 @@ module Herringbone
       end
     end
 
+    # @return [Integer] number of values held
     def size
       @indices ? @indices.size : @count
     end
 
+    # @return [Boolean] true when no values are held
     def empty? = size.zero?
 
+    # @return [Boolean] true while still dictionary-encoding (not yet switched to raw bytes)
     def dictionary? = !@indices.nil?
 
+    # Appends a value, switching to raw bytes when the dictionary limits are exceeded.
+    # @param value [String] bytes of the value; for FIXED_LEN_BYTE_ARRAY it must be +width+ bytes
+    #   long (not checked here)
+    # @return [ByteValues] self
     def <<(value)
       if @indices
         index = @dictionary[value]
@@ -61,16 +71,20 @@ module Herringbone
     end
 
     # Removes the last value
+    # @return [void]
     def pop
       truncate(size - 1) unless size.zero?
     end
 
     # Supports the `slice!(n..)` form used to roll back a failed row
+    # @param range [Range] endless range; values from +range.begin+ on are dropped
+    # @return [void]
     def slice!(range)
       truncate(range.begin)
     end
 
     # Approximate memory held, used to size row groups
+    # @return [Integer] estimated bytes
     def memory_bytes
       if @indices
         # Each distinct value is a String object plus a Hash entry
@@ -82,6 +96,11 @@ module Herringbone
 
     # Returns [:dictionary, values, indices] when the column is worth dictionary-encoding,
     # otherwise [:plain, values]
+    #
+    # Dictionary encoding is kept unless there are more than 16 values and more than about half
+    # of them are distinct.
+    # @return [Array(Symbol, Array<String>, Array<Integer>), Array(Symbol, Array<String>)]
+    #   distinct values and an index per value, or all values in order
     def materialize
       if @indices
         keys = @dictionary.keys
@@ -93,6 +112,9 @@ module Herringbone
 
     private
 
+    # Appends +value+ in raw-bytes mode, recording its length for BYTE_ARRAY.
+    # @param value [String] bytes of the value, in any encoding
+    # @return [ByteValues] self
     def append_bytes(value)
       if value.encoding == Encoding::BINARY || value.ascii_only?
         @bytes << value
@@ -106,6 +128,8 @@ module Herringbone
       self
     end
 
+    # Leaves dictionary mode, re-appending every value held so far as raw bytes.
+    # @return [void]
     def switch_to_bytes
       keys = @dictionary.keys
       indices = @indices
@@ -113,6 +137,9 @@ module Herringbone
       indices.each { |i| append_bytes(keys[i]) }
     end
 
+    # Drops all values after the first +n+.
+    # @param n [Integer] number of values to keep
+    # @return [void]
     def truncate(n)
       if @indices
         @indices.slice!(n..)
@@ -128,6 +155,8 @@ module Herringbone
       end
     end
 
+    # Splits the raw bytes back into one binary String per value.
+    # @return [Array<String>]
     def strings
       if @lengths
         pos = 0

data/lib/herringbone/codecs/lz4.rb

@@ -5,17 +5,26 @@ module Herringbone
     # Pure-Ruby LZ4: raw block format (Parquet LZ4_RAW), Hadoop-framed blocks
     # (Parquet's deprecated LZ4) and a decoder for the LZ4 frame format.
     module LZ4
+      # Raised for corrupt, truncated or unsupported LZ4 input.
       class Error < StandardError; end
 
+      # Shortest match a sequence can encode; the token's match length is stored minus this.
       MIN_MATCH = 4
       LAST_LITERALS = 5 # the last 5 bytes of a block are always literals
       MFLIMIT = 12      # the last match must start at least 12 bytes before the end
+      # Largest backward distance a 2-byte match offset can express.
       MAX_OFFSET = 65_535
+      # Size of the compressor's hash table, in bits (16384 entries).
       HASH_LOG = 14
+      # Right shift that keeps the top HASH_LOG bits of the 32-bit multiplicative hash.
       HASH_SHIFT = 32 - HASH_LOG
+      # Controls how fast the compressor skips ahead through input without matches (as in the reference).
       SKIP_STRENGTH = 6
+      # Magic number at the start of an LZ4 frame (little-endian).
       FRAME_MAGIC = 0x184D2204
+      # Hadoop block header: big-endian 32-bit uncompressed size and compressed size.
       HADOOP_PREFIX = 8
+      # Shorthand for Encoding::BINARY.
       BINARY = Encoding::BINARY
       # String#unpack1 accepts offset: since Ruby 3.1
       UNPACK_OFFSET = begin
@@ -28,6 +37,10 @@ module Herringbone
       module_function
 
       # Decompress a raw LZ4 block that must expand to exactly uncompressed_size bytes.
+      # @param input [String] raw LZ4 block
+      # @param uncompressed_size [Integer] exact decompressed size, from the page header
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the block is corrupt or does not decode to +uncompressed_size+ bytes
       def decompress_block(input, uncompressed_size)
         src = binary(input)
         out = String.new(capacity: uncompressed_size, encoding: BINARY)
@@ -40,6 +53,10 @@ module Herringbone
 
       # Parquet LZ4 (codec 5). Tries Hadoop framing, then the LZ4 frame format,
       # then a bare raw block (Arrow falls back hadoop -> raw; some writers emitted frames).
+      # @param input [String] compressed page data
+      # @param uncompressed_size [Integer] exact decompressed size, from the page header
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if none of the three layouts decodes
       def decompress_hadoop(input, uncompressed_size)
         src = binary(input)
         result = try_hadoop(src, uncompressed_size)
@@ -58,6 +75,10 @@ module Herringbone
 
       # Decode LZ4 frame format data (one or more frames, skippable frames ignored).
       # Checksums are skipped, not verified.
+      # @param input [String] one or more concatenated LZ4 frames
+      # @param max_size [Integer] upper bound on the decompressed size
+      # @return [String] decompressed bytes in ASCII-8BIT, possibly shorter than +max_size+
+      # @raise [Error] on bad magic, truncation, unsupported frame features or output over +max_size+
       def decompress_frame(input, max_size)
         src = binary(input)
         n = src.bytesize
@@ -79,6 +100,8 @@ module Herringbone
       end
 
       # Compress into a single raw LZ4 block.
+      # @param input [String] bytes to compress
+      # @return [String] raw LZ4 block in ASCII-8BIT
       def compress_block(input)
         src = binary(input)
         n = src.bytesize
@@ -121,7 +144,7 @@ module Herringbone
             # Emit the sequence: token, literal run, offset, extended match length
             lit_len = ip - anchor
             ml = len - MIN_MATCH
-            out << (((lit_len < 15 ? lit_len : 15) << 4) | (ml < 15 ? ml : 15))
+            out << ((((lit_len < 15) ? lit_len : 15) << 4) | ((ml < 15) ? ml : 15))
             write_length(out, lit_len - 15) if lit_len >= 15
             out << src.byteslice(anchor, lit_len) if lit_len > 0
             offset = ip - ref
@@ -144,6 +167,8 @@ module Herringbone
       end
 
       # Single Hadoop-framed block: [BE uncompressed size][BE compressed size][raw block]
+      # @param input [String] bytes to compress
+      # @return [String] Hadoop-framed LZ4 data in ASCII-8BIT
       def compress_hadoop(input)
         src = binary(input)
         block = compress_block(src)
@@ -152,18 +177,31 @@ module Herringbone
 
       # -- internals --
 
+      # @param str [String] input in any encoding
+      # @return [String] +str+ itself if already binary, otherwise a binary copy
       def binary(str)
-        str.encoding == BINARY ? str : str.b
+        (str.encoding == BINARY) ? str : str.b
       end
 
+      # @param src [String] binary input
+      # @param i [Integer] byte offset
+      # @return [Integer] unsigned little-endian 32-bit value at +i+
       def le32(src, i)
         src.byteslice(i, 4).unpack1("V")
       end
 
+      # Same as le32 but via getbyte, for Rubies without unpack1(offset:).
+      # @param src [String] binary input
+      # @param i [Integer] byte offset
+      # @return [Integer] unsigned little-endian 32-bit value at +i+
       def u32(src, i)
         src.getbyte(i) | (src.getbyte(i + 1) << 8) | (src.getbyte(i + 2) << 16) | (src.getbyte(i + 3) << 24)
       end
 
+      # Appends the extension of a literal or match length: 255-bytes followed by the remainder.
+      # @param out [String] binary output buffer, appended to
+      # @param len [Integer] length minus the 15 already stored in the token
+      # @return [String] +out+
       def write_length(out, len)
         if len >= 255
           out << ("\xFF".b * (len / 255))
@@ -172,14 +210,27 @@ module Herringbone
         out << len
       end
 
+      # Appends the final, literals-only sequence that terminates every block.
+      # @param out [String] binary output buffer, appended to
+      # @param src [String] binary input
+      # @param anchor [Integer] offset of the first pending literal in +src+
+      # @param lit_len [Integer] number of trailing literal bytes (may be zero)
+      # @return [String, nil] +out+, or nil when there are no literals
       def emit_last_literals(out, src, anchor, lit_len)
-        out << ((lit_len < 15 ? lit_len : 15) << 4)
+        out << (((lit_len < 15) ? lit_len : 15) << 4)
         write_length(out, lit_len - 15) if lit_len >= 15
         out << src.byteslice(anchor, lit_len) if lit_len > 0
       end
 
       # Decode one raw block from src[ip...iend], appending to out (which may already
       # hold earlier data that matches can reference). out may not grow beyond limit.
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the block in +src+
+      # @param iend [Integer] offset just past the block
+      # @param out [String] binary output buffer, appended to
+      # @param limit [Integer] maximum total byte size of +out+
+      # @return [Integer] input offset where decoding stopped
+      # @raise [Error] if the block is truncated, has an invalid offset or overflows +limit+
       def decode_block(src, ip, iend, out, limit)
         while ip < iend
           token = src.getbyte(ip)
@@ -237,6 +288,9 @@ module Herringbone
       end
 
       # Arrow-compatible Hadoop frame parsing; returns nil if the data does not fit the framing.
+      # @param src [String] binary input
+      # @param uncompressed_size [Integer] exact decompressed size expected over all blocks
+      # @return [String, nil] decompressed bytes, or nil if +src+ is not valid Hadoop-framed LZ4
       def try_hadoop(src, uncompressed_size)
         n = src.bytesize
         return nil if n < HADOOP_PREFIX
@@ -262,6 +316,15 @@ module Herringbone
         out
       end
 
+      # Decodes the descriptor and data blocks of one LZ4 frame (after its magic number),
+      # skipping content size and checksums.
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the frame descriptor (just past the magic)
+      # @param n [Integer] byte size of +src+
+      # @param out [String] binary output buffer, appended to
+      # @param limit [Integer] maximum total byte size of +out+
+      # @return [Integer] offset just past the frame
+      # @raise [Error] on truncation, an unsupported version or dictionary, or output over +limit+
       def decode_frame(src, ip, n, out, limit)
         raise Error, "Truncated LZ4 frame descriptor" if ip + 3 > n
         flg = src.getbyte(ip)