herringbone 0.2.0 → 0.3.0

data/lib/herringbone/encodings/delta.rb

@@ -6,20 +6,38 @@ module Herringbone
     module Delta
       module_function
 
+      # Values per block written by the encoder (the size parquet-mr and Arrow use).
       BLOCK_SIZE = 128
+      # Miniblocks per block written by the encoder, so 32 values per miniblock.
       MINIBLOCKS = 4
 
+      # @param n [Integer] zigzag-encoded unsigned integer
+      # @return [Integer] the signed integer it represents
       def zigzag_decode(n) = (n >> 1) ^ -(n & 1)
+
+      # @param n [Integer] signed integer
+      # @return [Integer] zigzag-encoded form (0, -1, 1, -2 map to 0, 1, 2, 3)
       def zigzag_encode(n) = n.negative? ? ((-n) << 1) - 1 : n << 1
 
       # Wraps an integer into the signed range of +bits+ bits
+      # @param v [Integer] integer to wrap
+      # @param bits [Integer] target width, 32 or 64
+      # @return [Integer] +v+ modulo 2^bits, as a two's complement signed integer
       def wrap(v, bits)
         half = 1 << (bits - 1)
         ((v + half) & ((1 << bits) - 1)) - half
       end
 
       # Decodes DELTA_BINARY_PACKED integers. +bits+ is 32 or 64 (for wraparound).
-      # Returns [values, new_pos]. If +count+ is nil, the total from the header is used.
+      # Returns [values, new_pos]. If +count+ is nil, the total from the header is used. A smaller
+      # +count+ decodes only that many values, but the returned offset is still the end of the
+      # whole encoded block, so data following it can be read from there.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the block header
+      # @param bits [Integer] integer width that deltas wrap around in, 32 or 64
+      # @param count [Integer, nil] maximum number of values to decode
+      # @return [Array(Array<Integer>, Integer)] the decoded values and the offset just past the encoded block
+      # @raise [FormatError] on an invalid header or miniblock bit width, or truncated data
       def decode_binary_packed(data, pos, bits = 64, count = nil)
         block_size, pos = RLE.read_uleb(data, pos)
         miniblocks, pos = RLE.read_uleb(data, pos)
@@ -28,34 +46,44 @@ module Herringbone
         raise FormatError, "Invalid DELTA_BINARY_PACKED header" if miniblocks.zero? || block_size % miniblocks != 0
         per_mini = block_size / miniblocks
         raise FormatError, "Invalid miniblock size #{per_mini}" if per_mini % 8 != 0
-        total = count if count && count < total
+        want = (count && count < total) ? count : total
         values = []
         return [values, pos] if total.zero?
         last = zigzag_decode(first)
-        values << last
+        values << last if want.positive?
         half = 1 << (bits - 1)
         mask = (1 << bits) - 1
-        while values.size < total
+        left = total - 1 # deltas still encoded, decoded or skipped
+        while left.positive?
           min_delta, pos = RLE.read_uleb(data, pos)
           min_delta = zigzag_decode(min_delta)
           widths = data.byteslice(pos, miniblocks).unpack("C*")
           pos += miniblocks
           widths.each do |w|
-            break if values.size >= total
+            # Miniblocks past the last value have a width byte but no body
+            break unless left.positive?
             raise FormatError, "Invalid delta bit width #{w}" if w > bits
-            deltas = RLE.unpack_bits(data, pos, per_mini, w)
-            pos += per_mini * w / 8
-            take = total - values.size
-            deltas = deltas.first(take) if take < per_mini
-            deltas.each do |d|
-              last = ((last + min_delta + d + half) & mask) - half
-              values << last
+            if values.size < want
+              deltas = RLE.unpack_bits(data, pos, per_mini, w)
+              take = want - values.size
+              deltas = deltas.first(take) if take < per_mini
+              deltas.each do |d|
+                last = ((last + min_delta + d + half) & mask) - half
+                values << last
+              end
             end
+            pos += per_mini * w / 8
+            left -= per_mini
           end
         end
         [values, pos]
       end
 
+      # Encodes integers with DELTA_BINARY_PACKED, using BLOCK_SIZE values per block in MINIBLOCKS
+      # miniblocks. Deltas wrap at +bits+ so INT32 columns never need more than 32-bit widths.
+      # @param values [Array<Integer>] signed integers that fit in +bits+ bits
+      # @param bits [Integer] physical integer width, 32 or 64
+      # @return [String] encoded bytes in ASCII-8BIT
       def encode_binary_packed(values, bits = 64)
         out = String.new(encoding: Encoding::BINARY)
         per_mini = BLOCK_SIZE / MINIBLOCKS
@@ -82,6 +110,12 @@ module Herringbone
         out
       end
 
+      # Decodes DELTA_LENGTH_BYTE_ARRAY: DELTA_BINARY_PACKED lengths followed by the concatenated bytes.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the lengths block
+      # @param count [Integer] number of values to decode
+      # @return [Array(Array<String>, Integer)] binary slices of +data+ and the offset just past them
+      # @raise [FormatError] if there are too few lengths or a value runs past the end of +data+
       def decode_length_byte_array(data, pos, count)
         lengths, pos = decode_binary_packed(data, pos, 32, count)
         raise FormatError, "DELTA_LENGTH_BYTE_ARRAY has #{lengths.size} lengths, need #{count}" if lengths.size < count
@@ -95,22 +129,35 @@ module Herringbone
         [out, pos]
       end
 
+      # Encodes byte strings with DELTA_LENGTH_BYTE_ARRAY.
+      # @param values [Array<String>] byte strings, in any encoding
+      # @return [String] encoded bytes in ASCII-8BIT
       def encode_length_byte_array(values)
         encode_binary_packed(values.map(&:bytesize), 32) << values.pack("a*" * values.size)
       end
 
+      # Decodes DELTA_BYTE_ARRAY (incremental encoding): DELTA_BINARY_PACKED prefix lengths, then
+      # the suffixes as DELTA_LENGTH_BYTE_ARRAY. Each value is the previous value's prefix plus its suffix.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the prefix lengths block
+      # @param count [Integer] number of values to decode
+      # @return [Array(Array<String>, Integer)] binary values and the offset just past them
+      # @raise [FormatError] if a prefix is negative or longer than the previous value, or the data is malformed
       def decode_byte_array(data, pos, count)
         prefixes, pos = decode_binary_packed(data, pos, 32, count)
         suffixes, pos = decode_length_byte_array(data, pos, count)
         prev = "".b
         out = Array.new(count) do |i|
           prefix = prefixes[i]
-          raise FormatError, "DELTA_BYTE_ARRAY prefix longer than previous value" if prefix > prev.bytesize
+          raise FormatError, "DELTA_BYTE_ARRAY prefix length #{prefix} out of range" if prefix > prev.bytesize || prefix.negative?
           prev = prefix.zero? ? suffixes[i] : prev.byteslice(0, prefix) + suffixes[i]
         end
         [out, pos]
       end
 
+      # Encodes byte strings with DELTA_BYTE_ARRAY, sharing the common byte prefix with the previous value.
+      # @param values [Array<String>] byte strings, in any encoding
+      # @return [String] encoded bytes in ASCII-8BIT
       def encode_byte_array(values)
         prev = "".b
         prefixes = []
@@ -132,6 +179,12 @@ module Herringbone
       module_function
 
       # Returns the value bytes re-interleaved into PLAIN layout
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the first stream
+      # @param count [Integer] number of values
+      # @param width [Integer] byte width of one value
+      # @return [Array(String, Integer)] PLAIN-layout bytes and the offset just past the streams
+      # @raise [FormatError] if fewer than count * width bytes remain
       def decode(data, pos, count, width)
         nbytes = count * width
         raise FormatError, "Truncated BYTE_STREAM_SPLIT data" if pos + nbytes > data.bytesize
@@ -139,6 +192,10 @@ module Herringbone
         [streams[0].zip(*streams[1..]).flatten.pack("C*"), pos + nbytes]
       end
 
+      # Splits PLAIN-layout fixed-width values into +width+ byte streams. Inverse of decode.
+      # @param plain [String] PLAIN-encoded values, a multiple of +width+ bytes
+      # @param width [Integer] byte width of one value
+      # @return [String] the concatenated streams in ASCII-8BIT
       def encode(plain, width)
         count = plain.bytesize / width
         bytes = plain.unpack("C*")

data/lib/herringbone/encodings/plain.rb

@@ -7,6 +7,7 @@ module Herringbone
     module Plain
       module_function
 
+      # Fixed-width numeric types: pack/unpack directive and byte width (all little-endian).
       FORMATS = {
         Format::Type::INT32 => ["l<", 4],
         Format::Type::INT64 => ["q<", 8],
@@ -16,6 +17,14 @@ module Herringbone
 
       # Decodes +count+ values of +type+ from +data+ starting at +pos+.
       # Returns [values, new_pos].
+      # INT96 values come back as [nanoseconds_of_day, julian_day] pairs.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the first value
+      # @param count [Integer] number of values to decode
+      # @param type [Integer] physical type, a Format::Type constant
+      # @param type_length [Integer, nil] byte width, required for FIXED_LEN_BYTE_ARRAY
+      # @return [Array(Array, Integer)] the decoded values and the offset just past them
+      # @raise [FormatError] if the data is truncated or the type is unknown
       def decode(data, pos, count, type, type_length = nil)
         case type
         when Format::Type::BOOLEAN
@@ -44,6 +53,12 @@ module Herringbone
         end
       end
 
+      # Decodes PLAIN BYTE_ARRAY values: each is a 4-byte little-endian length followed by the bytes.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the first length prefix
+      # @param count [Integer] number of values to decode
+      # @return [Array(Array<String>, Integer)] binary slices of +data+ and the offset just past them
+      # @raise [FormatError] if a length prefix or value runs past the end of +data+
       def decode_byte_arrays(data, pos, count)
         out = Array.new(count)
         size = data.bytesize
@@ -60,6 +75,13 @@ module Herringbone
         [out, pos]
       end
 
+      # Encodes values of a physical type with PLAIN. Inverse of decode; INT96 values are
+      # [nanoseconds_of_day, julian_day] pairs and booleans are packed LSB-first, 8 per byte.
+      # @param values [Array] values in their physical Ruby form, without nulls
+      # @param type [Integer] physical type, a Format::Type constant
+      # @param type_length [Integer, nil] byte width, required for FIXED_LEN_BYTE_ARRAY
+      # @return [String] encoded bytes in ASCII-8BIT
+      # @raise [EncodeError] if a FIXED_LEN_BYTE_ARRAY value has the wrong size or the type is unknown
       def encode(values, type, type_length = nil)
         case type
         when Format::Type::BOOLEAN

data/lib/herringbone/encodings/rle.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 module Herringbone
+  # Decoders and encoders for the Parquet value and level encodings: PLAIN, the RLE / bit-packed
+  # hybrid, the DELTA_* family and BYTE_STREAM_SPLIT. They work on binary Strings and byte offsets,
+  # and know nothing about pages or columns.
   module Encodings
     # Bit packing (LSB-first, as used by Parquet) and the RLE / bit-packed hybrid encoding
     # used for repetition/definition levels, dictionary indices and RLE booleans.
@@ -9,6 +12,11 @@ module Herringbone
 
       # Unpacks +count+ values of +width+ bits each, starting at byte +offset+ of +data+.
       # Missing trailing bytes are treated as zeroes.
+      # @param data [String] binary input
+      # @param offset [Integer] byte offset of the first packed value
+      # @param count [Integer] number of values to unpack
+      # @param width [Integer] bits per value, 0..64
+      # @return [Array<Integer>] +count+ unsigned values
       def unpack_bits(data, offset, count, width)
         return Array.new(count, 0) if width.zero?
         nbytes = (count * width + 7) / 8
@@ -22,7 +30,7 @@ module Herringbone
 
         # Pad to a whole number of 32-bit words plus one spare word, so reads never go out of range
         pad = (-chunk.bytesize % 4) + 4
-        words = (pad.zero? ? chunk : chunk + ("\0" * pad)).unpack("V*")
+        words = (chunk + ("\0" * pad)).unpack("V*")
         mask = (1 << width) - 1
         out = Array.new(count)
         bitpos = 0
@@ -41,6 +49,10 @@ module Herringbone
       end
 
       # Slow path for widths above 32 bits (used by DELTA_BINARY_PACKED with 64-bit values)
+      # @param chunk [String] packed bytes, starting at the first value
+      # @param count [Integer] number of values to unpack
+      # @param width [Integer] bits per value, 33..64
+      # @return [Array<Integer>] +count+ unsigned values
       def unpack_wide(chunk, count, width)
         mask = (1 << width) - 1
         out = Array.new(count)
@@ -61,6 +73,9 @@ module Herringbone
       end
 
       # Packs +values+ with +width+ bits each; the value count is padded up to a multiple of 8.
+      # @param values [Array<Integer>] non-negative integers that fit in +width+ bits
+      # @param width [Integer] bits per value
+      # @return [String] packed bytes in ASCII-8BIT, +width+ bytes per group of 8 values
       def pack_bits(values, width)
         return "".b if width.zero? || values.empty?
         n = values.size
@@ -93,6 +108,11 @@ module Herringbone
         out
       end
 
+      # Reads an unsigned LEB128 varint (hybrid run headers, DELTA_BINARY_PACKED headers).
+      # @param data [String] binary input
+      # @param pos [Integer] byte offset of the varint
+      # @return [Array(Integer, Integer)] the decoded value and the offset just past it
+      # @raise [FormatError] if the input ends inside the varint
       def read_uleb(data, pos)
         result = 0
         shift = 0
@@ -106,6 +126,10 @@ module Herringbone
         end
       end
 
+      # Appends +n+ as an unsigned LEB128 varint.
+      # @param out [String] binary output buffer, appended to
+      # @param n [Integer] non-negative integer to encode
+      # @return [String] +out+
       def write_uleb(out, n)
         while n >= 0x80
           out << ((n & 0x7F) | 0x80)
@@ -116,6 +140,13 @@ module Herringbone
 
       # Decodes the RLE/bit-packed hybrid from +data+ between +pos+ and +limit+,
       # returning exactly +count+ values (missing values are an error).
+      # @param data [String] binary input
+      # @param pos [Integer] byte offset of the first run header
+      # @param limit [Integer] byte offset just past the encoded data
+      # @param width [Integer] bits per value
+      # @param count [Integer] number of values to decode
+      # @return [Array<Integer>] exactly +count+ values
+      # @raise [FormatError] if the runs end before +count+ values were produced
       def decode_hybrid(data, pos, limit, width, count)
         out = []
         value_bytes = (width + 7) / 8
@@ -145,6 +176,10 @@ module Herringbone
 
       # Encodes +values+ with the RLE/bit-packed hybrid. Repeated runs of 8+ equal values
       # become RLE runs, everything else goes into bit-packed groups of 8.
+      # Output has no length prefix; callers that need one (levels in data page v1) add it.
+      # @param values [Array<Integer>] non-negative integers that fit in +width+ bits
+      # @param width [Integer] bits per value
+      # @return [String] encoded runs in ASCII-8BIT
       def encode_hybrid(values, width)
         out = String.new(encoding: Encoding::BINARY)
         n = values.size
@@ -181,6 +216,13 @@ module Herringbone
         out
       end
 
+      # Appends values[from...to] as one bit-packed run, zero-padded to whole groups of 8.
+      # @param out [String] binary output buffer, appended to
+      # @param values [Array<Integer>] all values being encoded
+      # @param from [Integer] index of the first literal value
+      # @param to [Integer] index just past the last literal value
+      # @param width [Integer] bits per value
+      # @return [String] +out+
       def flush_literals(out, values, from, to, width)
         count = to - from
         groups = (count + 7) / 8
@@ -188,13 +230,23 @@ module Herringbone
         out << pack_bits(values[from, count], width)
       end
 
+      # Number of bits needed to store values up to +max_value+ (0 for 0).
+      # @param max_value [Integer, nil] largest value to encode; nil counts as 0
+      # @return [Integer] bit width
       def bit_width(max_value)
         max_value.to_i.bit_length
       end
 
       # Legacy BIT_PACKED level encoding (deprecated): MSB-first bit order, no header.
+      # @param data [String] binary input
+      # @param pos [Integer] byte offset of the packed levels
+      # @param width [Integer] bits per value
+      # @param count [Integer] number of values to decode
+      # @return [Array<Integer>] +count+ levels
+      # @raise [FormatError] if +data+ ends before +count+ levels
       def decode_legacy_bit_packed(data, pos, width, count)
         nbytes = (count * width + 7) / 8
+        raise FormatError, "Truncated BIT_PACKED levels" if pos + nbytes > data.bytesize
         bits = data.byteslice(pos, nbytes).unpack1("B*")
         Array.new(count) { |i| bits[i * width, width].to_i(2) }
       end