herringbone 0.1.0 → 0.3.0

data/lib/herringbone/encodings/plain.rb

@@ -7,6 +7,7 @@ module Herringbone
     module Plain
       module_function
 
+      # Fixed-width numeric types: pack/unpack directive and byte width (all little-endian).
       FORMATS = {
         Format::Type::INT32 => ["l<", 4],
         Format::Type::INT64 => ["q<", 8],
@@ -16,43 +17,57 @@ module Herringbone
 
       # Decodes +count+ values of +type+ from +data+ starting at +pos+.
       # Returns [values, new_pos].
+      # INT96 values come back as [nanoseconds_of_day, julian_day] pairs.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the first value
+      # @param count [Integer] number of values to decode
+      # @param type [Integer] physical type, a Format::Type constant
+      # @param type_length [Integer, nil] byte width, required for FIXED_LEN_BYTE_ARRAY
+      # @return [Array(Array, Integer)] the decoded values and the offset just past them
+      # @raise [FormatError] if the data is truncated or the type is unknown
       def decode(data, pos, count, type, type_length = nil)
         case type
         when Format::Type::BOOLEAN
           nbytes = (count + 7) / 8
           bits = data.byteslice(pos, nbytes).unpack1("b*")
-          raise DecodeError, "Truncated BOOLEAN data" if bits.bytesize < count
+          raise FormatError, "Truncated BOOLEAN data" if bits.bytesize < count
           [Array.new(count) { |i| bits.getbyte(i) == 49 }, pos + nbytes]
         when Format::Type::INT32, Format::Type::INT64, Format::Type::FLOAT, Format::Type::DOUBLE
           fmt, width = FORMATS[type]
           nbytes = count * width
-          raise DecodeError, "Truncated PLAIN data" if pos + nbytes > data.bytesize
+          raise FormatError, "Truncated PLAIN data" if pos + nbytes > data.bytesize
           [data.byteslice(pos, nbytes).unpack("#{fmt}#{count}"), pos + nbytes]
         when Format::Type::INT96
           nbytes = count * 12
-          raise DecodeError, "Truncated INT96 data" if pos + nbytes > data.bytesize
+          raise FormatError, "Truncated INT96 data" if pos + nbytes > data.bytesize
           vals = data.byteslice(pos, nbytes).unpack("Q<L<" * count).each_slice(2).map { |nanos, day| [nanos, day] }
           [vals, pos + nbytes]
         when Format::Type::BYTE_ARRAY
           decode_byte_arrays(data, pos, count)
         when Format::Type::FIXED_LEN_BYTE_ARRAY
           nbytes = count * type_length
-          raise DecodeError, "Truncated FIXED_LEN_BYTE_ARRAY data" if pos + nbytes > data.bytesize
+          raise FormatError, "Truncated FIXED_LEN_BYTE_ARRAY data" if pos + nbytes > data.bytesize
           [Array.new(count) { |i| data.byteslice(pos + i * type_length, type_length) }, pos + nbytes]
         else
-          raise DecodeError, "Unknown physical type #{type}"
+          raise FormatError, "Unknown physical type #{type}"
         end
       end
 
+      # Decodes PLAIN BYTE_ARRAY values: each is a 4-byte little-endian length followed by the bytes.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the first length prefix
+      # @param count [Integer] number of values to decode
+      # @return [Array(Array<String>, Integer)] binary slices of +data+ and the offset just past them
+      # @raise [FormatError] if a length prefix or value runs past the end of +data+
       def decode_byte_arrays(data, pos, count)
         out = Array.new(count)
         size = data.bytesize
         i = 0
         while i < count
-          raise DecodeError, "Truncated BYTE_ARRAY data" if pos + 4 > size
+          raise FormatError, "Truncated BYTE_ARRAY data" if pos + 4 > size
           len = data.getbyte(pos) | (data.getbyte(pos + 1) << 8) | (data.getbyte(pos + 2) << 16) | (data.getbyte(pos + 3) << 24)
           pos += 4
-          raise DecodeError, "BYTE_ARRAY value overruns page" if pos + len > size
+          raise FormatError, "BYTE_ARRAY value overruns page" if pos + len > size
           out[i] = data.byteslice(pos, len)
           pos += len
           i += 1
@@ -60,6 +75,13 @@ module Herringbone
         [out, pos]
       end
 
+      # Encodes values of a physical type with PLAIN. Inverse of decode; INT96 values are
+      # [nanoseconds_of_day, julian_day] pairs and booleans are packed LSB-first, 8 per byte.
+      # @param values [Array] values in their physical Ruby form, without nulls
+      # @param type [Integer] physical type, a Format::Type constant
+      # @param type_length [Integer, nil] byte width, required for FIXED_LEN_BYTE_ARRAY
+      # @return [String] encoded bytes in ASCII-8BIT
+      # @raise [EncodeError] if a FIXED_LEN_BYTE_ARRAY value has the wrong size or the type is unknown
       def encode(values, type, type_length = nil)
         case type
         when Format::Type::BOOLEAN

data/lib/herringbone/encodings/rle.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 module Herringbone
+  # Decoders and encoders for the Parquet value and level encodings: PLAIN, the RLE / bit-packed
+  # hybrid, the DELTA_* family and BYTE_STREAM_SPLIT. They work on binary Strings and byte offsets,
+  # and know nothing about pages or columns.
   module Encodings
     # Bit packing (LSB-first, as used by Parquet) and the RLE / bit-packed hybrid encoding
     # used for repetition/definition levels, dictionary indices and RLE booleans.
@@ -9,6 +12,11 @@ module Herringbone
 
       # Unpacks +count+ values of +width+ bits each, starting at byte +offset+ of +data+.
       # Missing trailing bytes are treated as zeroes.
+      # @param data [String] binary input
+      # @param offset [Integer] byte offset of the first packed value
+      # @param count [Integer] number of values to unpack
+      # @param width [Integer] bits per value, 0..64
+      # @return [Array<Integer>] +count+ unsigned values
       def unpack_bits(data, offset, count, width)
         return Array.new(count, 0) if width.zero?
         nbytes = (count * width + 7) / 8
@@ -22,7 +30,7 @@ module Herringbone
 
         # Pad to a whole number of 32-bit words plus one spare word, so reads never go out of range
         pad = (-chunk.bytesize % 4) + 4
-        words = (pad.zero? ? chunk : chunk + ("\0" * pad)).unpack("V*")
+        words = (chunk + ("\0" * pad)).unpack("V*")
         mask = (1 << width) - 1
         out = Array.new(count)
         bitpos = 0
@@ -41,6 +49,10 @@ module Herringbone
       end
 
       # Slow path for widths above 32 bits (used by DELTA_BINARY_PACKED with 64-bit values)
+      # @param chunk [String] packed bytes, starting at the first value
+      # @param count [Integer] number of values to unpack
+      # @param width [Integer] bits per value, 33..64
+      # @return [Array<Integer>] +count+ unsigned values
       def unpack_wide(chunk, count, width)
         mask = (1 << width) - 1
         out = Array.new(count)
@@ -61,6 +73,9 @@ module Herringbone
       end
 
       # Packs +values+ with +width+ bits each; the value count is padded up to a multiple of 8.
+      # @param values [Array<Integer>] non-negative integers that fit in +width+ bits
+      # @param width [Integer] bits per value
+      # @return [String] packed bytes in ASCII-8BIT, +width+ bytes per group of 8 values
       def pack_bits(values, width)
         return "".b if width.zero? || values.empty?
         n = values.size
@@ -93,12 +108,17 @@ module Herringbone
         out
       end
 
+      # Reads an unsigned LEB128 varint (hybrid run headers, DELTA_BINARY_PACKED headers).
+      # @param data [String] binary input
+      # @param pos [Integer] byte offset of the varint
+      # @return [Array(Integer, Integer)] the decoded value and the offset just past it
+      # @raise [FormatError] if the input ends inside the varint
       def read_uleb(data, pos)
         result = 0
         shift = 0
         while true
           b = data.getbyte(pos)
-          raise DecodeError, "Truncated varint" unless b
+          raise FormatError, "Truncated varint" unless b
           pos += 1
           result |= (b & 0x7F) << shift
           return [result, pos] if b < 0x80
@@ -106,6 +126,10 @@ module Herringbone
         end
       end
 
+      # Appends +n+ as an unsigned LEB128 varint.
+      # @param out [String] binary output buffer, appended to
+      # @param n [Integer] non-negative integer to encode
+      # @return [String] +out+
       def write_uleb(out, n)
         while n >= 0x80
           out << ((n & 0x7F) | 0x80)
@@ -116,11 +140,18 @@ module Herringbone
 
       # Decodes the RLE/bit-packed hybrid from +data+ between +pos+ and +limit+,
       # returning exactly +count+ values (missing values are an error).
+      # @param data [String] binary input
+      # @param pos [Integer] byte offset of the first run header
+      # @param limit [Integer] byte offset just past the encoded data
+      # @param width [Integer] bits per value
+      # @param count [Integer] number of values to decode
+      # @return [Array<Integer>] exactly +count+ values
+      # @raise [FormatError] if the runs end before +count+ values were produced
       def decode_hybrid(data, pos, limit, width, count)
         out = []
         value_bytes = (width + 7) / 8
         while out.size < count
-          raise DecodeError, "RLE data exhausted (#{out.size}/#{count} values)" if pos >= limit
+          raise FormatError, "RLE data exhausted (#{out.size}/#{count} values)" if pos >= limit
           header, pos = read_uleb(data, pos)
           if header & 1 == 1
             groups = header >> 1
@@ -145,6 +176,10 @@ module Herringbone
 
       # Encodes +values+ with the RLE/bit-packed hybrid. Repeated runs of 8+ equal values
       # become RLE runs, everything else goes into bit-packed groups of 8.
+      # Output has no length prefix; callers that need one (levels in data page v1) add it.
+      # @param values [Array<Integer>] non-negative integers that fit in +width+ bits
+      # @param width [Integer] bits per value
+      # @return [String] encoded runs in ASCII-8BIT
       def encode_hybrid(values, width)
         out = String.new(encoding: Encoding::BINARY)
         n = values.size
@@ -181,6 +216,13 @@ module Herringbone
         out
       end
 
+      # Appends values[from...to] as one bit-packed run, zero-padded to whole groups of 8.
+      # @param out [String] binary output buffer, appended to
+      # @param values [Array<Integer>] all values being encoded
+      # @param from [Integer] index of the first literal value
+      # @param to [Integer] index just past the last literal value
+      # @param width [Integer] bits per value
+      # @return [String] +out+
       def flush_literals(out, values, from, to, width)
         count = to - from
         groups = (count + 7) / 8
@@ -188,13 +230,23 @@ module Herringbone
         out << pack_bits(values[from, count], width)
       end
 
+      # Number of bits needed to store values up to +max_value+ (0 for 0).
+      # @param max_value [Integer, nil] largest value to encode; nil counts as 0
+      # @return [Integer] bit width
       def bit_width(max_value)
         max_value.to_i.bit_length
       end
 
       # Legacy BIT_PACKED level encoding (deprecated): MSB-first bit order, no header.
+      # @param data [String] binary input
+      # @param pos [Integer] byte offset of the packed levels
+      # @param width [Integer] bits per value
+      # @param count [Integer] number of values to decode
+      # @return [Array<Integer>] +count+ levels
+      # @raise [FormatError] if +data+ ends before +count+ levels
       def decode_legacy_bit_packed(data, pos, width, count)
         nbytes = (count * width + 7) / 8
+        raise FormatError, "Truncated BIT_PACKED levels" if pos + nbytes > data.bytesize
         bits = data.byteslice(pos, nbytes).unpack1("B*")
         Array.new(count) { |i| bits[i * width, width].to_i(2) }
       end

data/lib/herringbone/format.rb

@@ -4,92 +4,163 @@ module Herringbone
   # Parquet file metadata structures, mirroring parquet.thrift.
   # Enums are plain i32 on the wire; constants below give them names.
   module Format
+    # Physical storage types (+Type+ enum in parquet.thrift).
     module Type
+      # Single bit, bit-packed in PLAIN encoding
       BOOLEAN = 0
+      # 32-bit signed little-endian integer
       INT32 = 1
+      # 64-bit signed little-endian integer
       INT64 = 2
+      # 96-bit legacy timestamp (nanoseconds of day + Julian day), deprecated by the spec
       INT96 = 3
+      # IEEE 754 single precision
       FLOAT = 4
+      # IEEE 754 double precision
       DOUBLE = 5
+      # Variable-length bytes, length-prefixed in PLAIN encoding
       BYTE_ARRAY = 6
+      # Bytes of the fixed length given by +SchemaElement#type_length+
       FIXED_LEN_BYTE_ARRAY = 7
+      # Constant name for each enum value
       NAMES = constants.to_h { |c| [const_get(c), c] }.freeze
     end
 
+    # Legacy type annotations (+ConvertedType+ enum), superseded by LogicalType but still
+    # written alongside it for older readers.
     module ConvertedType
+      # UTF-8 encoded BYTE_ARRAY
       UTF8 = 0
+      # Group annotated as a map
       MAP = 1
+      # Repeated key/value group inside a MAP
       MAP_KEY_VALUE = 2
+      # Group annotated as a list
       LIST = 3
+      # BYTE_ARRAY holding an enum label
       ENUM = 4
+      # Decimal with scale and precision from the SchemaElement
       DECIMAL = 5
+      # INT32 days since the Unix epoch
       DATE = 6
+      # INT32 milliseconds since midnight
       TIME_MILLIS = 7
+      # INT64 microseconds since midnight
       TIME_MICROS = 8
+      # INT64 milliseconds since the Unix epoch, UTC
       TIMESTAMP_MILLIS = 9
+      # INT64 microseconds since the Unix epoch, UTC
       TIMESTAMP_MICROS = 10
+      # Unsigned 8-bit integer stored in INT32
       UINT_8 = 11
+      # Unsigned 16-bit integer stored in INT32
       UINT_16 = 12
+      # Unsigned 32-bit integer stored in INT32
       UINT_32 = 13
+      # Unsigned 64-bit integer stored in INT64
       UINT_64 = 14
+      # Signed 8-bit integer stored in INT32
       INT_8 = 15
+      # Signed 16-bit integer stored in INT32
       INT_16 = 16
+      # Signed 32-bit integer stored in INT32
       INT_32 = 17
+      # Signed 64-bit integer stored in INT64
       INT_64 = 18
+      # UTF-8 JSON document in a BYTE_ARRAY
       JSON = 19
+      # BSON document in a BYTE_ARRAY
       BSON = 20
+      # 12-byte FIXED_LEN_BYTE_ARRAY of months, days and milliseconds
       INTERVAL = 21
+      # Constant name for each enum value
       NAMES = constants.to_h { |c| [const_get(c), c] }.freeze
     end
 
+    # Field repetition (+FieldRepetitionType+ enum).
     module Repetition
+      # Exactly one value
       REQUIRED = 0
+      # Zero or one value
       OPTIONAL = 1
+      # Zero or more values
       REPEATED = 2
+      # Constant name for each enum value
       NAMES = constants.to_h { |c| [const_get(c), c] }.freeze
     end
 
+    # Value and level encodings (+Encoding+ enum). Value 1 (GROUP_VAR_INT) was never used.
     module Encoding
+      # Values back to back in their natural binary form
       PLAIN = 0
+      # Deprecated name for a dictionary-encoded page (and a PLAIN dictionary page) in format v1
       PLAIN_DICTIONARY = 2
+      # RLE / bit-packing hybrid, used for levels and booleans
       RLE = 3
+      # Deprecated bit-packed levels
       BIT_PACKED = 4
+      # Delta encoding for INT32/INT64
       DELTA_BINARY_PACKED = 5
+      # Delta-encoded lengths followed by the concatenated bytes
       DELTA_LENGTH_BYTE_ARRAY = 6
+      # Incremental (prefix-shared) encoding for byte arrays
       DELTA_BYTE_ARRAY = 7
+      # Dictionary indices in RLE / bit-packing hybrid form
       RLE_DICTIONARY = 8
+      # Bytes of each value split into separate streams
       BYTE_STREAM_SPLIT = 9
+      # Constant name for each enum value
       NAMES = constants.to_h { |c| [const_get(c), c] }.freeze
     end
 
+    # Page compression codecs (+CompressionCodec+ enum).
     module Codec
+      # No compression
       UNCOMPRESSED = 0
+      # Raw Snappy
       SNAPPY = 1
+      # Gzip (deflate with gzip header)
       GZIP = 2
+      # LZO
       LZO = 3
+      # Brotli
       BROTLI = 4
+      # Deprecated Hadoop-framed LZ4
       LZ4 = 5
+      # Zstandard
       ZSTD = 6
+      # LZ4 block format without framing
       LZ4_RAW = 7
+      # Constant name for each enum value
       NAMES = constants.to_h { |c| [const_get(c), c] }.freeze
     end
 
+    # Page types (+PageType+ enum).
     module PageType
+      # Data page, format v1
       DATA_PAGE = 0
+      # Index page (never written in practice)
       INDEX_PAGE = 1
+      # Dictionary page, precedes the data pages of a column chunk
       DICTIONARY_PAGE = 2
+      # Data page, format v2 (levels stored uncompressed ahead of the values)
       DATA_PAGE_V2 = 3
+      # Constant name for each enum value
       NAMES = constants.to_h { |c| [const_get(c), c] }.freeze
     end
 
+    # Short alias for the struct base class used by every definition below
     S = Thrift::Struct
 
+    # Byte and level-histogram statistics for a page or column chunk.
     class SizeStatistics < S
       field 1, :unencoded_byte_array_data_bytes, :i64
       field 2, :repetition_level_histogram, [:list, :i64]
       field 3, :definition_level_histogram, [:list, :i64]
     end
 
+    # Min/max and count statistics for a page or column chunk. +max+/+min+ are the deprecated
+    # signed-order fields; +max_value+/+min_value+ use the column's sort order.
     class Statistics < S
       field 1, :max, :binary
       field 2, :min, :binary
@@ -102,40 +173,77 @@ module Herringbone
     end
 
     # Empty marker structs used inside unions
+
+    # STRING logical type.
     class StringType < S; end
+
+    # UUID logical type (16-byte FIXED_LEN_BYTE_ARRAY).
     class UUIDType < S; end
+
+    # MAP logical type.
     class MapType < S; end
+
+    # LIST logical type.
     class ListType < S; end
+
+    # ENUM logical type.
     class EnumType < S; end
+
+    # DATE logical type.
     class DateType < S; end
+
+    # FLOAT16 logical type (2-byte FIXED_LEN_BYTE_ARRAY, little-endian half precision).
     class Float16Type < S; end
+
+    # UNKNOWN logical type: the column is always null.
     class NullType < S; end
+
+    # JSON logical type.
     class JsonType < S; end
+
+    # BSON logical type.
     class BsonType < S; end
+
+    # Millisecond member of the TimeUnit union.
     class MilliSeconds < S; end
+
+    # Microsecond member of the TimeUnit union.
     class MicroSeconds < S; end
+
+    # Nanosecond member of the TimeUnit union.
     class NanoSeconds < S; end
+
+    # ColumnOrder member: values are ordered according to their (logical) type.
     class TypeDefinedOrder < S; end
+
+    # Header of an index page; has no fields in the spec.
     class IndexPageHeader < S; end
 
+    # VARIANT logical type.
     class VariantType < S
       field 1, :specification_version, :byte
     end
 
+    # DECIMAL logical type: unscaled integer value divided by 10 to the power of +scale+.
     class DecimalType < S
       field 1, :scale, :i32
       field 2, :precision, :i32
     end
 
+    # Union of time units for TIME and TIMESTAMP; exactly one member is set.
     class TimeUnit < S
       field 1, :millis, MilliSeconds
       field 2, :micros, MicroSeconds
       field 3, :nanos, NanoSeconds
 
+      # @return [TimeUnit] a unit with the +millis+ member set
       def self.millis = new(millis: MilliSeconds.new)
+      # @return [TimeUnit] a unit with the +micros+ member set
       def self.micros = new(micros: MicroSeconds.new)
+      # @return [TimeUnit] a unit with the +nanos+ member set
       def self.nanos = new(nanos: NanoSeconds.new)
 
+      # @return [Symbol, nil] +:millis+, +:micros+ or +:nanos+, or nil when no member is set
       def to_sym
         if millis then :millis
         elsif micros then :micros
@@ -144,21 +252,26 @@ module Herringbone
       end
     end
 
+    # TIMESTAMP logical type.
     class TimestampType < S
       field 1, :is_adjusted_to_utc, :bool
       field 2, :unit, TimeUnit
     end
 
+    # TIME logical type.
     class TimeType < S
       field 1, :is_adjusted_to_utc, :bool
       field 2, :unit, TimeUnit
     end
 
+    # INTEGER logical type (bit width 8, 16, 32 or 64; signed or unsigned).
     class IntType < S
       field 1, :bit_width, :byte
       field 2, :is_signed, :bool
     end
 
+    # Union of logical type annotations; exactly one member is set. Field id 9 is reserved
+    # (for INTERVAL) in parquet.thrift.
     class LogicalType < S
       field 1, :string, StringType
       field 2, :map, MapType
@@ -177,6 +290,8 @@ module Herringbone
       field 16, :variant, VariantType
 
       # Returns [kind_symbol, payload] for whichever union member is set
+      # @return [Array(Symbol, Thrift::Struct), nil] member name and its struct, or nil when
+      #   no member is set
       def kind
         self.class.fields.each do |f|
           v = instance_variable_get(f.ivar)
@@ -186,6 +301,8 @@ module Herringbone
       end
     end
 
+    # One node of the schema, flattened depth-first into +FileMetaData#schema+. Groups carry
+    # +num_children+; leaves carry the physical +type+.
     class SchemaElement < S
       field 1, :type, :i32
       field 2, :type_length, :i32
@@ -199,6 +316,7 @@ module Herringbone
       field 10, :logical_type, LogicalType
     end
 
+    # Header of a v1 data page: levels and values share one compressed block.
     class DataPageHeader < S
       field 1, :num_values, :i32
       field 2, :encoding, :i32
@@ -207,12 +325,15 @@ module Herringbone
       field 5, :statistics, Statistics
     end
 
+    # Header of a dictionary page.
     class DictionaryPageHeader < S
       field 1, :num_values, :i32
       field 2, :encoding, :i32
       field 3, :is_sorted, :bool
     end
 
+    # Header of a v2 data page: levels are stored uncompressed before the (optionally
+    # compressed) values, with their byte lengths given here.
     class DataPageHeaderV2 < S
       field 1, :num_values, :i32
       field 2, :num_nulls, :i32
@@ -224,6 +345,7 @@ module Herringbone
       field 8, :statistics, Statistics
     end
 
+    # Header preceding every page; +type+ (a PageType) says which sub-header is set.
     class PageHeader < S
       field 1, :type, :i32
       field 2, :uncompressed_page_size, :i32
@@ -235,23 +357,27 @@ module Herringbone
       field 8, :data_page_header_v2, DataPageHeaderV2
     end
 
+    # Application-defined key/value metadata entry.
     class KeyValue < S
       field 1, :key, :string
       field 2, :value, :string
     end
 
+    # Sort order of one column within a row group.
     class SortingColumn < S
       field 1, :column_idx, :i32
       field 2, :descending, :bool
       field 3, :nulls_first, :bool
     end
 
+    # Number of pages of a given type and encoding in a column chunk.
     class PageEncodingStats < S
       field 1, :page_type, :i32
       field 2, :encoding, :i32
       field 3, :count, :i32
     end
 
+    # Metadata of a column chunk: where its pages are, how they are encoded and compressed.
     class ColumnMetaData < S
       field 1, :type, :i32
       field 2, :encodings, [:list, :i32]
@@ -271,6 +397,8 @@ module Herringbone
       field 16, :size_statistics, SizeStatistics
     end
 
+    # One column of a row group, plus the locations of its page index. Field id 8
+    # (crypto_metadata) is not supported.
     class ColumnChunk < S
       field 1, :file_path, :string
       field 2, :file_offset, :i64
@@ -282,6 +410,7 @@ module Herringbone
       field 9, :encrypted_column_metadata, :binary
     end
 
+    # A horizontal slice of the file, holding one ColumnChunk per leaf column.
     class RowGroup < S
       field 1, :columns, [:list, ColumnChunk]
       field 2, :total_byte_size, :i64
@@ -292,10 +421,50 @@ module Herringbone
       field 7, :ordinal, :i16
     end
 
+    # Ordering of the per-page min/max values in a ColumnIndex (+BoundaryOrder+ enum).
+    module BoundaryOrder
+      # No particular order
+      UNORDERED = 0
+      # Min and max values both non-decreasing from page to page
+      ASCENDING = 1
+      # Min and max values both non-increasing from page to page
+      DESCENDING = 2
+      # Constant name for each enum value
+      NAMES = constants.to_h { |c| [const_get(c), c] }.freeze
+    end
+
+    # Page index structures (stored between the row groups and the footer)
+
+    # Location of one data page within the file.
+    class PageLocation < S
+      field 1, :offset, :i64
+      field 2, :compressed_page_size, :i32
+      field 3, :first_row_index, :i64
+    end
+
+    # Offset index of a column chunk: the location of each of its data pages.
+    class OffsetIndex < S
+      field 1, :page_locations, [:list, PageLocation]
+      field 2, :unencoded_byte_array_data_bytes, [:list, :i64]
+    end
+
+    # Column index of a column chunk: per-page min/max values and null information.
+    class ColumnIndex < S
+      field 1, :null_pages, [:list, :bool]
+      field 2, :min_values, [:list, :binary]
+      field 3, :max_values, [:list, :binary]
+      field 4, :boundary_order, :i32
+      field 5, :null_counts, [:list, :i64]
+      field 6, :repetition_level_histograms, [:list, :i64]
+      field 7, :definition_level_histograms, [:list, :i64]
+    end
+
+    # Union describing how min/max statistics of a column are ordered.
     class ColumnOrder < S
       field 1, :type_order, TypeDefinedOrder
     end
 
+    # The file footer: schema, row groups and file-level metadata.
     class FileMetaData < S
       field 1, :version, :i32
       field 2, :schema, [:list, SchemaElement]
@@ -305,5 +474,39 @@ module Herringbone
       field 6, :created_by, :string
       field 7, :column_orders, [:list, ColumnOrder]
     end
+
+    # Bloom filters (BloomFilter.md). Each union has a single member, an empty struct.
+
+    # Split block bloom filter algorithm.
+    class SplitBlockAlgorithm < S; end
+
+    # XXH64 hash with seed 0.
+    class XxHash < S; end
+
+    # Bloom filter bitset stored uncompressed.
+    class BloomFilterUncompressed < S; end
+
+    # Union of bloom filter algorithms.
+    class BloomFilterAlgorithm < S
+      field 1, :block, SplitBlockAlgorithm
+    end
+
+    # Union of hash functions used to feed the bloom filter.
+    class BloomFilterHash < S
+      field 1, :xxhash, XxHash
+    end
+
+    # Union of bloom filter compressions.
+    class BloomFilterCompression < S
+      field 1, :uncompressed, BloomFilterUncompressed
+    end
+
+    # Header preceding the bitset of a column chunk's bloom filter.
+    class BloomFilterHeader < S
+      field 1, :num_bytes, :i32
+      field 2, :algorithm, BloomFilterAlgorithm
+      field 3, :hash_function, BloomFilterHash # "hash" in parquet.thrift; renamed to keep Object#hash
+      field 4, :compression, BloomFilterCompression
+    end
   end
 end