herringbone 0.1.0 → 0.2.0

data/lib/herringbone/bloom_filter.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+module Herringbone
+  # A Parquet Split Block Bloom Filter (parquet-format BloomFilter.md).
+  #
+  # The bitset is made of 32-byte blocks of eight 32-bit words. A value is hashed with XXH64 over
+  # the PLAIN encoding of its physical value (without the length prefix for BYTE_ARRAY); the upper
+  # 32 bits of the hash pick a block, and the lower 32 bits set one bit in each of its words.
+  #
+  # A filter answers "definitely not in the column chunk" or "maybe": +might_contain?+ never
+  # returns false for a value that was inserted. Values are given as Ruby values and converted
+  # like the writer converts them (the column's encoder), so a Date, Time, BigDecimal or UUID
+  # String hashes the same bytes as the stored value. Nulls are never in a bloom filter.
+  # The writer builds them (bloom_filters: option) and reads with where: consult them.
+  class BloomFilter
+    SALT = [0x47b6137b, 0x44974d91, 0x8824ad5b, 0xa2b7289d, 0x705495c7, 0x2df1424b, 0x9efc4947, 0x5c6bfb31].freeze
+    S0, S1, S2, S3, S4, S5, S6, S7 = SALT
+    # Low 16 bits of the salts: (lo * salt) mod 2**32 is computed as
+    # (lo & 0xFFFF) * salt + ((lo >> 16) * (salt & 0xFFFF) << 16), which stays a Fixnum
+    L0, L1, L2, L3, L4, L5, L6, L7 = SALT.map { |s| s & 0xFFFF }
+    BLOCK_BYTES = 32
+    MIN_BYTES = 32
+    MAX_BYTES = 128 * 1024 * 1024
+    # Default cap for filters sized by the writer (as in parquet-mr)
+    DEFAULT_MAX_BYTES = 1024 * 1024
+    DEFAULT_FPP = 0.01
+    M32 = 0xFFFF_FFFF
+    M64 = 0xFFFF_FFFF_FFFF_FFFF
+    T = Format::Type
+    # Physical types a bloom filter can be built for (the spec does not define BOOLEAN hashing)
+    TYPES = [T::INT32, T::INT64, T::INT96, T::FLOAT, T::DOUBLE, T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY].freeze
+
+    # Bitset size in bytes for +ndv+ distinct values at false positive probability +fpp+, per the
+    # spec's formula (m = -8 * ndv / ln(1 - fpp ** (1/8)) bits), rounded up to a power of two
+    # and clamped to MIN_BYTES..max_bytes
+    def self.optimal_num_bytes(ndv, fpp = DEFAULT_FPP, max_bytes: DEFAULT_MAX_BYTES)
+      fpp = Float(fpp)
+      raise ArgumentError, "fpp must be between 0 and 1, got #{fpp}" unless fpp > 0 && fpp < 1
+      max_bytes = [[Integer(max_bytes), MAX_BYTES].min, MIN_BYTES].max
+      max_bytes = 1 << (max_bytes.bit_length - 1) # a power of two
+      ndv = [Integer(ndv), 1].max
+      bits = -8.0 * ndv / Math.log(1 - fpp**(1.0 / 8))
+      bytes = (bits / 8).ceil
+      return max_bytes if bytes >= max_bytes
+      return MIN_BYTES if bytes <= MIN_BYTES
+      1 << (bytes - 1).bit_length
+    end
+
+    # XXH64 of the PLAIN encoding of a physical value of +type+ (as the column encoders produce it)
+    def self.hash_physical(value, type)
+      case type
+      when T::INT32 then XXHash.xxh64_u32(value)
+      when T::INT64 then XXHash.xxh64_u64(value)
+      when T::FLOAT then XXHash.xxh64_u32([value].pack("e").unpack1("L<"))
+      when T::DOUBLE then XXHash.xxh64_u64([value].pack("E").unpack1("Q<"))
+      when T::INT96 then XXHash.xxh64(value.pack("Q<L<"))
+      when T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY then XXHash.xxh64(value)
+      else raise UnsupportedError, "Bloom filters are not defined for #{T::NAMES.fetch(type, type)} values"
+      end
+    end
+
+    # Hashes of many physical values of +type+, converting them in bulk. With +distinct+, each
+    # distinct physical value is hashed once (floats are compared by their bytes, so -0.0 and 0.0,
+    # or NaNs with different payloads, stay apart as they hash differently).
+    def self.hash_physical_all(values, type, distinct: false)
+      case type
+      when T::INT32 then XXHash.xxh64_u32_all(distinct ? values.uniq : values)
+      when T::INT64 then XXHash.xxh64_u64_all(distinct ? values.uniq : values)
+      when T::FLOAT
+        words = values.pack("e*").unpack("L<*")
+        XXHash.xxh64_u32_all(distinct ? words.uniq! || words : words)
+      when T::DOUBLE
+        lanes = values.pack("E*").unpack("Q<*")
+        XXHash.xxh64_u64_all(distinct ? lanes.uniq! || lanes : lanes)
+      when T::INT96 then XXHash.xxh64_all((distinct ? values.uniq : values).map { |v| v.pack("Q<L<") })
+      when T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY then XXHash.xxh64_all(distinct ? values.uniq : values)
+      else raise UnsupportedError, "Bloom filters are not defined for #{T::NAMES.fetch(type, type)} values"
+      end
+    end
+
+    # Reads a filter (header and bitset) from +buf+ at +pos+. Returns nil for algorithms, hashes
+    # or compressions this implementation does not know.
+    def self.decode(buf, pos = 0, column: nil)
+      reader = Thrift::Reader.new(buf, pos)
+      header = reader.read_struct(Format::BloomFilterHeader)
+      return nil unless supported_header?(header)
+      bitset = buf.byteslice(reader.pos, header.num_bytes)
+      raise FormatError, "Truncated bloom filter" if bitset.nil? || bitset.bytesize != header.num_bytes
+      new(bitset: bitset, column: column)
+    end
+
+    def self.supported_header?(header)
+      n = header.num_bytes
+      n.is_a?(Integer) && n >= BLOCK_BYTES && (n % BLOCK_BYTES).zero? && n <= MAX_BYTES &&
+        header.algorithm&.block && header.hash_function&.xxhash &&
+        (header.compression.nil? || header.compression.uncompressed)
+    end
+
+    attr_reader :column
+
+    # A filter of +num_bytes+ (a multiple of 32, normally a power of two), or one using an existing
+    # +bitset+ String. With a +column+ (a Schema::Column), values are Ruby values converted with
+    # the column's encoder; without one, values must be Strings and their bytes are hashed.
+    def initialize(num_bytes = nil, bitset: nil, column: nil)
+      if bitset
+        @words = bitset.unpack("V*")
+      else
+        num_bytes = Integer(num_bytes || MIN_BYTES)
+        unless num_bytes >= BLOCK_BYTES && (num_bytes % BLOCK_BYTES).zero? && num_bytes <= MAX_BYTES
+          raise ArgumentError, "Bloom filter size must be a multiple of 32 bytes up to 128MB, got #{num_bytes}"
+        end
+        @words = Array.new(num_bytes / 4, 0)
+      end
+      @num_blocks = @words.size / 8
+      raise ArgumentError, "Bloom filter bitset must be a multiple of 32 bytes" if @num_blocks.zero? || @words.size % 8 != 0
+      @column = column
+      if column && !TYPES.include?(column.type)
+        raise UnsupportedError, "Bloom filters are not supported for #{T::NAMES[column.type]} column #{column.dotted_path}"
+      end
+    end
+
+    def num_bytes = @words.size * 4
+
+    def insert(value)
+      insert_hash(hash_of(value))
+      self
+    end
+
+    def might_contain?(value)
+      might_contain_hash?(hash_of(value))
+    end
+
+    # XXH64 hash the filter uses for a Ruby +value+
+    def hash_of(value)
+      raise ArgumentError, "Nulls are not recorded in bloom filters" if value.nil?
+      if @column
+        begin
+          physical = @column.encoder.call(value)
+        rescue ArgumentError, TypeError, NoMethodError, RangeError, EncodeError => e
+          raise ArgumentError, "#{value.inspect} is not a valid value for #{@column.dotted_path}: #{e.message}"
+        end
+        self.class.hash_physical(physical, @column.type)
+      else
+        raise ArgumentError, "Without a column, bloom filter values must be Strings, got #{value.class}" unless value.is_a?(String)
+        XXHash.xxh64(value)
+      end
+    end
+
+    def insert_hash(h)
+      i = (((h >> 32) * @num_blocks) >> 32) << 3
+      x0 = h & 0xFFFF
+      x1 = (h >> 16) & 0xFFFF
+      w = @words
+      w[i] |= 1 << (((x0 * S0 + (x1 * L0 << 16)) & M32) >> 27)
+      w[i + 1] |= 1 << (((x0 * S1 + (x1 * L1 << 16)) & M32) >> 27)
+      w[i + 2] |= 1 << (((x0 * S2 + (x1 * L2 << 16)) & M32) >> 27)
+      w[i + 3] |= 1 << (((x0 * S3 + (x1 * L3 << 16)) & M32) >> 27)
+      w[i + 4] |= 1 << (((x0 * S4 + (x1 * L4 << 16)) & M32) >> 27)
+      w[i + 5] |= 1 << (((x0 * S5 + (x1 * L5 << 16)) & M32) >> 27)
+      w[i + 6] |= 1 << (((x0 * S6 + (x1 * L6 << 16)) & M32) >> 27)
+      w[i + 7] |= 1 << (((x0 * S7 + (x1 * L7 << 16)) & M32) >> 27)
+      self
+    end
+
+    BITS = Array.new(32) { |i| 1 << i }.freeze
+
+    # Inserts many hashes at once: faster than insert_hash, as the hashes (mostly Bignums) are
+    # split into 32-bit halves in bulk and the rest is Fixnum arithmetic
+    def insert_hashes(hashes)
+      halves = hashes.pack("Q<*").unpack("V*")
+      w = @words
+      num_blocks = @num_blocks
+      bit = BITS
+      j = 0
+      n = halves.size
+      while j < n
+        lo = halves[j]
+        i = halves[j + 1] * num_blocks / 4_294_967_296 * 8
+        x0 = lo & 0xFFFF
+        x1 = lo / 65_536
+        w[i] |= bit[(x0 * S0 + x1 * L0 * 65536) / 134_217_728 & 31]
+        w[i + 1] |= bit[(x0 * S1 + x1 * L1 * 65536) / 134_217_728 & 31]
+        w[i + 2] |= bit[(x0 * S2 + x1 * L2 * 65536) / 134_217_728 & 31]
+        w[i + 3] |= bit[(x0 * S3 + x1 * L3 * 65536) / 134_217_728 & 31]
+        w[i + 4] |= bit[(x0 * S4 + x1 * L4 * 65536) / 134_217_728 & 31]
+        w[i + 5] |= bit[(x0 * S5 + x1 * L5 * 65536) / 134_217_728 & 31]
+        w[i + 6] |= bit[(x0 * S6 + x1 * L6 * 65536) / 134_217_728 & 31]
+        w[i + 7] |= bit[(x0 * S7 + x1 * L7 * 65536) / 134_217_728 & 31]
+        j += 2
+      end
+      self
+    end
+
+    def might_contain_hash?(h)
+      i = (((h >> 32) * @num_blocks) >> 32) << 3
+      x0 = h & 0xFFFF
+      x1 = (h >> 16) & 0xFFFF
+      w = @words
+      w[i][(((x0 * S0 + (x1 * L0 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 1][(((x0 * S1 + (x1 * L1 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 2][(((x0 * S2 + (x1 * L2 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 3][(((x0 * S3 + (x1 * L3 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 4][(((x0 * S4 + (x1 * L4 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 5][(((x0 * S5 + (x1 * L5 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 6][(((x0 * S6 + (x1 * L6 << 16)) & M32) >> 27)] == 1 &&
+        w[i + 7][(((x0 * S7 + (x1 * L7 << 16)) & M32) >> 27)] == 1
+    end
+
+    # The raw bitset (little-endian 32-bit words)
+    def bitset = @words.pack("V*")
+
+    # Header and bitset, as stored in a Parquet file
+    def encode
+      header.encode << bitset
+    end
+
+    def inspect
+      "#<#{self.class.name} #{num_bytes} bytes#{" for #{@column.dotted_path}" if @column}>"
+    end
+
+    private
+
+    def header
+      Format::BloomFilterHeader.new(
+        num_bytes: num_bytes,
+        algorithm: Format::BloomFilterAlgorithm.new(block: Format::SplitBlockAlgorithm.new),
+        hash_function: Format::BloomFilterHash.new(xxhash: Format::XxHash.new),
+        compression: Format::BloomFilterCompression.new(uncompressed: Format::BloomFilterUncompressed.new)
+      )
+    end
+  end
+
+  class Reader
+    # Internal (used by reads with where:): the bloom filter of a column chunk. +column+ is a
+    # dotted path ("a.b"), an Array path or a Schema::Column. Returns a BloomFilter, or nil when
+    # the chunk has none (or one of an unknown kind).
+    def bloom_filter(row_group_index, column)
+      col = bloom_filter_column(column)
+      rg = row_groups.fetch(row_group_index) { raise IndexError, "No row group #{row_group_index}" }
+      meta = rg.columns.fetch(col.index).meta_data
+      offset = meta&.bloom_filter_offset
+      return nil unless offset
+      length = meta.bloom_filter_length
+      @io.seek(offset)
+      if length
+        buf = @io.read(length)
+        raise FormatError, "Truncated bloom filter" if buf.nil? || buf.bytesize < length
+        return BloomFilter.decode(buf, column: col)
+      end
+      # Without a length (older writers), read the header first, then the bitset it announces
+      head = @io.read(256) || "".b
+      reader = Thrift::Reader.new(head)
+      header = reader.read_struct(Format::BloomFilterHeader)
+      return nil unless BloomFilter.supported_header?(header)
+      @io.seek(offset + reader.pos)
+      bitset = @io.read(header.num_bytes)
+      raise FormatError, "Truncated bloom filter" if bitset.nil? || bitset.bytesize != header.num_bytes
+      BloomFilter.new(bitset: bitset, column: col)
+    end
+
+    private
+
+    def bloom_filter_column(column)
+      return column if column.is_a?(Schema::Column)
+      col = schema.column(column.is_a?(Array) ? column.map(&:to_s) : column.to_s)
+      raise ArgumentError, "No column #{column.inspect}" unless col
+      col
+    end
+  end
+end

data/lib/herringbone/compression.rb

@@ -2,25 +2,82 @@
 
 require "zlib"
 require "stringio"
-require "zstd-ruby"
-require "brotli"
 
 module Herringbone
-  # Dispatches page (de)compression by Parquet codec id. Snappy and LZ4 are pure Ruby,
-  # GZIP uses zlib, ZSTD and Brotli use the zstd-ruby and brotli gems.
+  # Raised when a file uses (or a writer asks for) a codec whose library is not installed
+  class MissingCodecError < UnsupportedError
+    attr_reader :codec, :gem_name
+
+    def initialize(codec, gem_name, load_error)
+      @codec = codec
+      @gem_name = gem_name
+      super("#{codec} compression needs the \"#{gem_name}\" gem, which could not be loaded " \
+        "(#{load_error.message}). Add `gem \"#{gem_name}\"` to your Gemfile to use #{codec}.")
+    end
+  end
+
+  # Dispatches page (de)compression by Parquet codec id. Snappy and LZ4 are pure Ruby and GZIP
+  # uses zlib, so those always work. ZSTD and Brotli come from optional gems (zstd-ruby, brotli),
+  # which are required on first use; if they are missing, MissingCodecError says what to add.
   module Compression
     module_function
 
-    CODECS_BY_NAME = {
-      none: Format::Codec::UNCOMPRESSED, uncompressed: Format::Codec::UNCOMPRESSED,
-      snappy: Format::Codec::SNAPPY, gzip: Format::Codec::GZIP, brotli: Format::Codec::BROTLI,
-      lz4: Format::Codec::LZ4_RAW, lz4_raw: Format::Codec::LZ4_RAW, lz4_hadoop: Format::Codec::LZ4,
-      zstd: Format::Codec::ZSTD
+    # Codecs backed by optional native gems: codec id => [name, gem, require path, constant]
+    LIBRARIES = {
+      Format::Codec::ZSTD => ["ZSTD", "zstd-ruby", "zstd-ruby", :Zstd],
+      Format::Codec::BROTLI => ["Brotli", "brotli", "brotli", :Brotli]
+    }.freeze
+
+    @libraries = {}
+    @library_mutex = Mutex.new
+
+    # The library module for a codec backed by an optional gem, requiring it on first use
+    def library(codec)
+      @libraries.fetch(codec) do
+        @library_mutex.synchronize do
+          @libraries.fetch(codec) do
+            name, gem_name, path, const = LIBRARIES.fetch(codec)
+            begin
+              require_library(path)
+            rescue LoadError => e
+              raise MissingCodecError.new(name, gem_name, e)
+            end
+            @libraries[codec] = Object.const_get(const)
+          end
+        end
+      end
+    end
+
+    def require_library(path)
+      require path
+    end
+
+    # Raises MissingCodecError (or UnsupportedError) unless +codec+ can be used
+    def ensure_available!(codec)
+      codec = codec_id(codec)
+      return library(codec) if LIBRARIES.key?(codec)
+      return if SUPPORTED.include?(codec)
+      raise UnsupportedError, "#{Format::Codec::NAMES[codec] || codec} compression is not supported"
+    end
+
+    SUPPORTED = [
+      Format::Codec::UNCOMPRESSED, Format::Codec::SNAPPY, Format::Codec::GZIP,
+      Format::Codec::LZ4_RAW, Format::Codec::LZ4
+    ].freeze
+
+    # Codec id => name, as given to the writer's compression: option and listed by Herringbone.codecs
+    NAMES = {
+      Format::Codec::UNCOMPRESSED => :none, Format::Codec::SNAPPY => :snappy, Format::Codec::GZIP => :gzip,
+      Format::Codec::LZ4_RAW => :lz4, Format::Codec::LZ4 => :lz4_hadoop, Format::Codec::ZSTD => :zstd,
+      Format::Codec::BROTLI => :brotli, Format::Codec::LZO => :lzo
     }.freeze
+    CODECS_BY_NAME = NAMES.invert.freeze
 
     def codec_id(name)
       return name if name.is_a?(Integer)
-      CODECS_BY_NAME.fetch(name.to_s.downcase.to_sym) { raise ArgumentError, "Unknown compression codec #{name.inspect}" }
+      CODECS_BY_NAME.fetch(name.to_s.downcase.to_sym) do
+        raise ArgumentError, "Unknown compression codec #{name.inspect}, expected one of #{NAMES.values.map(&:inspect).join(", ")}"
+      end
     end
 
     def decompress(codec, data, uncompressed_size)
@@ -31,14 +88,14 @@ module Herringbone
       when Format::Codec::GZIP then gunzip(data)
       when Format::Codec::LZ4_RAW then Codecs::LZ4.decompress_block(data, uncompressed_size)
       when Format::Codec::LZ4 then Codecs::LZ4.decompress_hadoop(data, uncompressed_size)
-      when Format::Codec::ZSTD then ::Zstd.decompress(data)
-      when Format::Codec::BROTLI then ::Brotli.inflate(data)
+      when Format::Codec::ZSTD then library(codec).decompress(data)
+      when Format::Codec::BROTLI then library(codec).inflate(data)
       else
         raise UnsupportedError, "Unsupported compression codec #{Format::Codec::NAMES[codec] || codec}"
       end
       out = out.b
       if out.bytesize != uncompressed_size
-        raise DecodeError, "Decompressed #{out.bytesize} bytes, expected #{uncompressed_size}"
+        raise FormatError, "Decompressed #{out.bytesize} bytes, expected #{uncompressed_size}"
       end
       out
     end
@@ -50,8 +107,8 @@ module Herringbone
       when Format::Codec::GZIP then Zlib.gzip(data)
       when Format::Codec::LZ4_RAW then Codecs::LZ4.compress_block(data)
       when Format::Codec::LZ4 then Codecs::LZ4.compress_hadoop(data)
-      when Format::Codec::ZSTD then ::Zstd.compress(data)
-      when Format::Codec::BROTLI then ::Brotli.deflate(data)
+      when Format::Codec::ZSTD then library(codec).compress(data)
+      when Format::Codec::BROTLI then library(codec).deflate(data)
       else
         raise UnsupportedError, "Unsupported compression codec #{Format::Codec::NAMES[codec] || codec}"
       end.b
@@ -61,7 +118,7 @@ module Herringbone
     def gunzip(data)
       io = StringIO.new(data)
       out = String.new(encoding: Encoding::BINARY)
-      loop do
+      while true
         gz = Zlib::GzipReader.new(io)
         out << gz.read
         unused = gz.unused

data/lib/herringbone/encodings/delta.rb

@@ -25,9 +25,9 @@ module Herringbone
         miniblocks, pos = RLE.read_uleb(data, pos)
         total, pos = RLE.read_uleb(data, pos)
         first, pos = RLE.read_uleb(data, pos)
-        raise DecodeError, "Invalid DELTA_BINARY_PACKED header" if miniblocks.zero? || block_size % miniblocks != 0
+        raise FormatError, "Invalid DELTA_BINARY_PACKED header" if miniblocks.zero? || block_size % miniblocks != 0
         per_mini = block_size / miniblocks
-        raise DecodeError, "Invalid miniblock size #{per_mini}" if per_mini % 8 != 0
+        raise FormatError, "Invalid miniblock size #{per_mini}" if per_mini % 8 != 0
         total = count if count && count < total
         values = []
         return [values, pos] if total.zero?
@@ -42,7 +42,7 @@ module Herringbone
           pos += miniblocks
           widths.each do |w|
             break if values.size >= total
-            raise DecodeError, "Invalid delta bit width #{w}" if w > bits
+            raise FormatError, "Invalid delta bit width #{w}" if w > bits
             deltas = RLE.unpack_bits(data, pos, per_mini, w)
             pos += per_mini * w / 8
             take = total - values.size
@@ -84,11 +84,11 @@ module Herringbone
 
       def decode_length_byte_array(data, pos, count)
         lengths, pos = decode_binary_packed(data, pos, 32, count)
-        raise DecodeError, "DELTA_LENGTH_BYTE_ARRAY has #{lengths.size} lengths, need #{count}" if lengths.size < count
+        raise FormatError, "DELTA_LENGTH_BYTE_ARRAY has #{lengths.size} lengths, need #{count}" if lengths.size < count
         out = Array.new(count)
         size = data.bytesize
         lengths.each_with_index do |len, i|
-          raise DecodeError, "DELTA_LENGTH_BYTE_ARRAY value overruns page" if len.negative? || pos + len > size
+          raise FormatError, "DELTA_LENGTH_BYTE_ARRAY value overruns page" if len.negative? || pos + len > size
           out[i] = data.byteslice(pos, len)
           pos += len
         end
@@ -105,7 +105,7 @@ module Herringbone
         prev = "".b
         out = Array.new(count) do |i|
           prefix = prefixes[i]
-          raise DecodeError, "DELTA_BYTE_ARRAY prefix longer than previous value" if prefix > prev.bytesize
+          raise FormatError, "DELTA_BYTE_ARRAY prefix longer than previous value" if prefix > prev.bytesize
           prev = prefix.zero? ? suffixes[i] : prev.byteslice(0, prefix) + suffixes[i]
         end
         [out, pos]
@@ -134,7 +134,7 @@ module Herringbone
       # Returns the value bytes re-interleaved into PLAIN layout
       def decode(data, pos, count, width)
         nbytes = count * width
-        raise DecodeError, "Truncated BYTE_STREAM_SPLIT data" if pos + nbytes > data.bytesize
+        raise FormatError, "Truncated BYTE_STREAM_SPLIT data" if pos + nbytes > data.bytesize
         streams = Array.new(width) { |k| data.byteslice(pos + k * count, count).unpack("C*") }
         [streams[0].zip(*streams[1..]).flatten.pack("C*"), pos + nbytes]
       end

data/lib/herringbone/encodings/plain.rb

@@ -21,26 +21,26 @@ module Herringbone
         when Format::Type::BOOLEAN
           nbytes = (count + 7) / 8
           bits = data.byteslice(pos, nbytes).unpack1("b*")
-          raise DecodeError, "Truncated BOOLEAN data" if bits.bytesize < count
+          raise FormatError, "Truncated BOOLEAN data" if bits.bytesize < count
           [Array.new(count) { |i| bits.getbyte(i) == 49 }, pos + nbytes]
         when Format::Type::INT32, Format::Type::INT64, Format::Type::FLOAT, Format::Type::DOUBLE
           fmt, width = FORMATS[type]
           nbytes = count * width
-          raise DecodeError, "Truncated PLAIN data" if pos + nbytes > data.bytesize
+          raise FormatError, "Truncated PLAIN data" if pos + nbytes > data.bytesize
           [data.byteslice(pos, nbytes).unpack("#{fmt}#{count}"), pos + nbytes]
         when Format::Type::INT96
           nbytes = count * 12
-          raise DecodeError, "Truncated INT96 data" if pos + nbytes > data.bytesize
+          raise FormatError, "Truncated INT96 data" if pos + nbytes > data.bytesize
           vals = data.byteslice(pos, nbytes).unpack("Q<L<" * count).each_slice(2).map { |nanos, day| [nanos, day] }
           [vals, pos + nbytes]
         when Format::Type::BYTE_ARRAY
           decode_byte_arrays(data, pos, count)
         when Format::Type::FIXED_LEN_BYTE_ARRAY
           nbytes = count * type_length
-          raise DecodeError, "Truncated FIXED_LEN_BYTE_ARRAY data" if pos + nbytes > data.bytesize
+          raise FormatError, "Truncated FIXED_LEN_BYTE_ARRAY data" if pos + nbytes > data.bytesize
           [Array.new(count) { |i| data.byteslice(pos + i * type_length, type_length) }, pos + nbytes]
         else
-          raise DecodeError, "Unknown physical type #{type}"
+          raise FormatError, "Unknown physical type #{type}"
         end
       end
 
@@ -49,10 +49,10 @@ module Herringbone
         size = data.bytesize
         i = 0
         while i < count
-          raise DecodeError, "Truncated BYTE_ARRAY data" if pos + 4 > size
+          raise FormatError, "Truncated BYTE_ARRAY data" if pos + 4 > size
           len = data.getbyte(pos) | (data.getbyte(pos + 1) << 8) | (data.getbyte(pos + 2) << 16) | (data.getbyte(pos + 3) << 24)
           pos += 4
-          raise DecodeError, "BYTE_ARRAY value overruns page" if pos + len > size
+          raise FormatError, "BYTE_ARRAY value overruns page" if pos + len > size
           out[i] = data.byteslice(pos, len)
           pos += len
           i += 1

data/lib/herringbone/encodings/rle.rb

@@ -98,7 +98,7 @@ module Herringbone
         shift = 0
         while true
           b = data.getbyte(pos)
-          raise DecodeError, "Truncated varint" unless b
+          raise FormatError, "Truncated varint" unless b
           pos += 1
           result |= (b & 0x7F) << shift
           return [result, pos] if b < 0x80
@@ -120,7 +120,7 @@ module Herringbone
         out = []
         value_bytes = (width + 7) / 8
         while out.size < count
-          raise DecodeError, "RLE data exhausted (#{out.size}/#{count} values)" if pos >= limit
+          raise FormatError, "RLE data exhausted (#{out.size}/#{count} values)" if pos >= limit
           header, pos = read_uleb(data, pos)
           if header & 1 == 1
             groups = header >> 1

data/lib/herringbone/format.rb

@@ -292,6 +292,35 @@ module Herringbone
       field 7, :ordinal, :i16
     end
 
+    module BoundaryOrder
+      UNORDERED = 0
+      ASCENDING = 1
+      DESCENDING = 2
+      NAMES = constants.to_h { |c| [const_get(c), c] }.freeze
+    end
+
+    # Page index structures (stored between the row groups and the footer)
+    class PageLocation < S
+      field 1, :offset, :i64
+      field 2, :compressed_page_size, :i32
+      field 3, :first_row_index, :i64
+    end
+
+    class OffsetIndex < S
+      field 1, :page_locations, [:list, PageLocation]
+      field 2, :unencoded_byte_array_data_bytes, [:list, :i64]
+    end
+
+    class ColumnIndex < S
+      field 1, :null_pages, [:list, :bool]
+      field 2, :min_values, [:list, :binary]
+      field 3, :max_values, [:list, :binary]
+      field 4, :boundary_order, :i32
+      field 5, :null_counts, [:list, :i64]
+      field 6, :repetition_level_histograms, [:list, :i64]
+      field 7, :definition_level_histograms, [:list, :i64]
+    end
+
     class ColumnOrder < S
       field 1, :type_order, TypeDefinedOrder
     end
@@ -305,5 +334,29 @@ module Herringbone
       field 6, :created_by, :string
       field 7, :column_orders, [:list, ColumnOrder]
     end
+
+    # Bloom filters (BloomFilter.md). Each union has a single member, an empty struct.
+    class SplitBlockAlgorithm < S; end
+    class XxHash < S; end
+    class BloomFilterUncompressed < S; end
+
+    class BloomFilterAlgorithm < S
+      field 1, :block, SplitBlockAlgorithm
+    end
+
+    class BloomFilterHash < S
+      field 1, :xxhash, XxHash
+    end
+
+    class BloomFilterCompression < S
+      field 1, :uncompressed, BloomFilterUncompressed
+    end
+
+    class BloomFilterHeader < S
+      field 1, :num_bytes, :i32
+      field 2, :algorithm, BloomFilterAlgorithm
+      field 3, :hash_function, BloomFilterHash # "hash" in parquet.thrift; renamed to keep Object#hash
+      field 4, :compression, BloomFilterCompression
+    end
   end
 end