herringbone 0.1.0 → 0.3.0

data/lib/herringbone/writer.rb

@@ -8,38 +8,54 @@ module Herringbone
   #     string :name
   #     list :tags, :string
   #   end
-  #   Herringbone::Writer.open("out.parquet", schema) do |w|
-  #     w << { "id" => 1, "name" => "one", "tags" => ["a", "b"] }
-  #     w << [2, "two", []]           # Arrays are taken in schema order
-  #     w << order                    # objects responding to #attributes (ActiveRecord) or #to_h
+  #   File.open("out.parquet", "wb") do |file|
+  #     Herringbone::Writer.open(file, schema) do |w|
+  #       w << { "id" => 1, "name" => "one", "tags" => ["a", "b"] }
+  #       w << [2, "two", []]           # Arrays are taken in schema order
+  #       w << order                    # objects responding to #attributes (ActiveRecord) or #to_h
+  #     end
   #   end
   #
-  # +schema+ is a Herringbone::Schema or a Hash spec (see Schema.define). The target is a path or an IO;
-  # paths are written to a temporary file next to the target and renamed into place on close, so a
-  # failed write never leaves a truncated file behind.
+  # The output is any IO that responds to #write (a File, StringIO, Tempfile, socket, pipe...); it
+  # is written sequentially and never seeked, rewound or closed by the writer; only #write is
+  # required, its return value is ignored, and #binmode and #flush are called when available.
+  # Herringbone does not open files by path.
   #
   # Options:
-  #   compression:     :zstd (default), :snappy, :gzip, :lz4 (LZ4_RAW), :brotli, :none
+  #   compression:     :snappy (default), :zstd, :gzip, :lz4 (LZ4_RAW), :lz4_hadoop, :brotli, :none
+  #                    (:zstd and :brotli need the zstd-ruby / brotli gems)
   #   row_group_bytes: flush a row group once the buffered values take roughly this much memory
   #                    (default 16MB). This bounds memory use while writing.
-  #   row_group_size:  also flush after this many rows (default: no row limit)
-  #   page_size:       approximate uncompressed data page size in bytes (default 1MB)
+  #   row_group_rows:  also flush after this many rows (default: no row limit)
+  #   page_bytes:      approximate uncompressed data page size (default 1MB)
+  #   page_rows:       at most this many rows per data page (default 20_000), which keeps the page
+  #                    index selective
   #   data_page_version: 1 (default) or 2
   #   dictionary:      true/false, or an Array of column paths to dictionary-encode
   #   encodings:       { "path.to.column" => :delta_binary_packed, ... } for non-dictionary pages
-  #   statistics:      write min/max/null_count statistics (default true)
   #   metadata:        Hash of String => String key/value metadata for the footer
+  #   bloom_filters:   write split block bloom filters: true (every column that supports them),
+  #                    an Array of column paths, or { "path" => true | { ndv:, fpp:, max_bytes: } }.
+  #                    Without ndv: the distinct values of each row group are counted. fpp defaults
+  #                    to 0.01 and max_bytes to 1MB. Filters are written after each row group.
+  #
+  # Statistics and page indexes (ColumnIndex/OffsetIndex) are always written.
   class Writer
-    MAGIC = "PAR1"
+    # Marker at the start and end of every Parquet file
+    MAGIC = "PAR1".b.freeze
+    # Shorthand for Format::Type (physical types)
     T = Format::Type
+    # Shorthand for Format::Encoding
     E = Format::Encoding
 
+    # Names accepted in the +encodings:+ option => Parquet encoding id
     ENCODING_NAMES = {
       plain: E::PLAIN, rle: E::RLE, delta_binary_packed: E::DELTA_BINARY_PACKED,
       delta_length_byte_array: E::DELTA_LENGTH_BYTE_ARRAY, delta_byte_array: E::DELTA_BYTE_ARRAY,
       byte_stream_split: E::BYTE_STREAM_SPLIT
     }.freeze
 
+    # Encoding id => physical types it may be used for, per the Parquet encodings spec
     VALID_ENCODINGS = {
       E::PLAIN => T::NAMES.keys,
       E::RLE => [T::BOOLEAN],
@@ -49,20 +65,47 @@ module Herringbone
       E::BYTE_STREAM_SPLIT => [T::INT32, T::INT64, T::FLOAT, T::DOUBLE, T::FIXED_LEN_BYTE_ARRAY]
     }.freeze
 
+    # Largest dictionary build_dictionary keeps; above it the chunk is written without a dictionary.
+    # Byte-array columns are dictionary-encoded by ByteValues, which applies its own
+    # ByteValues::MAX_DICTIONARY_BYTES as values arrive.
     MAX_DICTIONARY_BYTES = 1024 * 1024
     # The row group byte size is first estimated after this many rows, then after every row group
     ESTIMATE_AFTER_ROWS = 1000
 
+    # @return [Schema] schema the rows are written with
     attr_reader :schema
 
-    # Opens a writer on a path or an IO. With a block, the file is finished when the block returns
-    # (or discarded if it raises) and the block's value is returned; without one, call #close.
-    def self.open(target, schema, **options)
-      writer = new(target, schema, **options)
+    # Opens a writer on +io+. With a block, the file is finished (footer written) when the block
+    # returns and the block's value is returned; if the block raises, the writer is aborted and no
+    # footer is written. Without a block, call #close to finish. The IO is never closed.
+    #
+    # @param io [IO, #write] destination
+    # @param schema [Schema] schema of the rows
+    # @param options [Hash{Symbol => Object}] see the class description and #initialize
+    # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
+    # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
+    # @option options [Integer] :page_bytes (1MB) approximate uncompressed data page size
+    # @option options [Integer] :page_rows (20_000) maximum rows per data page
+    # @option options [Integer] :data_page_version (1) 1 or 2
+    # @option options [Boolean, Array<String>] :dictionary (true) dictionary-encode all eligible columns,
+    #   none, or only the listed dotted column paths
+    # @option options [Hash{String => Symbol}] :encodings ({}) dotted column path => value encoding
+    #   for non-dictionary pages
+    # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
+    # @option options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
+    #   columns to write split block bloom filters for
+    # @yield [writer] the open writer
+    # @yieldparam writer [Writer] writer to append rows to
+    # @yieldreturn [Object] returned by open
+    # @return [Writer, Object] the writer without a block, the block's value with one
+    # @raise [ArgumentError] for an invalid IO, schema or option
+    def self.open(io, schema, **options)
+      writer = new(io, schema, **options)
       return writer unless block_given?
       begin
         result = yield writer
-      rescue Exception # rubocop:disable Lint/RescueException -- also discard on Interrupt
+      rescue Exception # rubocop:disable Lint/RescueException -- also abort on Interrupt
         writer.abort
         raise
       end
@@ -70,35 +113,72 @@ module Herringbone
       result
     end
 
-    def initialize(target, schema, compression: :zstd, row_group_bytes: 16 * 1024 * 1024, row_group_size: nil,
-      page_size: 1024 * 1024, data_page_version: 1, dictionary: true, encodings: {}, statistics: true, metadata: {})
-      @schema = Schema.coerce(schema)
-      schema = @schema
+    # Validates the options and writes the leading magic bytes to +io+.
+    #
+    # @param io [IO, #write] destination; switched to binmode when it supports that
+    # @param schema [Schema] schema of the rows
+    # @param compression [Symbol, Integer] codec name (see Herringbone.codecs) or Format::Codec id
+    # @param row_group_bytes [Integer] approximate buffered size that triggers a row group
+    # @param row_group_rows [Integer, nil] also flush a row group after this many rows
+    # @param page_bytes [Integer] approximate uncompressed data page size
+    # @param page_rows [Integer] maximum level entries per data page (pages of repeated columns
+    #   extend to the next row start)
+    # @param data_page_version [Integer] 1 or 2
+    # @param dictionary [Boolean, Array<String>] true for every column except BOOLEAN, FLOAT, DOUBLE
+    #   and those listed in +encodings+; false for none; or the dotted paths of the columns to encode
+    # @param encodings [Hash{String, Symbol => Symbol, Integer}] dotted column path => value encoding
+    #   (a name from ENCODING_NAMES or an encoding id) for non-dictionary pages
+    # @param metadata [Hash{#to_s => #to_s}] key/value metadata for the footer
+    # @param bloom_filters [Boolean, Array<String>, Hash{String => Boolean, Hash}, nil] see the class
+    #   description
+    # @raise [ArgumentError] for an invalid IO, schema or option
+    # @raise [MissingCodecError] when the codec's optional gem cannot be loaded
+    # @raise [UnsupportedError] when the codec is not supported
+    def initialize(io, schema, compression: :snappy, row_group_bytes: 16 * 1024 * 1024, row_group_rows: nil,
+      page_bytes: 1024 * 1024, page_rows: 20_000, data_page_version: 1, dictionary: true, encodings: {},
+      metadata: {}, bloom_filters: nil)
+      raise ArgumentError, "Expected a Herringbone::Schema, got #{schema.class}" unless schema.is_a?(Schema)
+      @schema = schema
       @codec = Compression.codec_id(compression)
+      # Fail before creating any file if the codec's library is missing
+      Compression.ensure_available!(@codec)
       @row_group_bytes = Integer(row_group_bytes)
-      @row_group_size = row_group_size && Integer(row_group_size)
-      @row_limit = @row_group_size || ESTIMATE_AFTER_ROWS
-      @page_size = Integer(page_size)
+      @row_group_rows = row_group_rows && Integer(row_group_rows)
+      @row_limit = @row_group_rows || ESTIMATE_AFTER_ROWS
+      @page_bytes = Integer(page_bytes)
+      @page_rows = Integer(page_rows)
+      raise ArgumentError, "page_rows must be positive" unless @page_rows.positive?
+      @page_indexes = [] # [ColumnChunk, ColumnIndex or nil, OffsetIndex] per written column chunk
       @data_page_version = Integer(data_page_version)
       raise ArgumentError, "data_page_version must be 1 or 2" unless [1, 2].include?(@data_page_version)
       @dictionary = dictionary
       @encodings = encodings.to_h { |path, enc| [path.to_s, encoding_id(path, enc)] }
-      unknown = @encodings.keys - schema.columns.map(&:dotted_path)
+      columns = schema.columns.to_h { |c| [c.dotted_path, c] }
+      unknown = @encodings.keys - columns.keys
       raise ArgumentError, "encodings: no such column #{unknown.join(", ")}" unless unknown.empty?
-      @statistics = statistics
+      @encodings.each { |path, enc| check_encoding!(columns[path], enc) }
       @metadata = metadata
+      @bloom_filters = bloom_filter_config(bloom_filters)
+      @pending_bloom_filters = [] # [ColumnMetaData, BloomFilter] for the row group being written
       @row_groups = []
       @total_rows = 0
       @pos = 0
       @closed = false
       @bytes_per_row = nil
-      open_target(target)
+      @io = check_io!(io)
       write_raw(MAGIC)
       reset_buffers
     end
 
     # Appends a row: a Hash keyed by top-level field names (Strings or Symbols), an Array of values
-    # in schema order, or an object responding to #attributes (ActiveRecord) or #to_h (Struct, Data)
+    # in schema order, or an object responding to #attributes (ActiveRecord) or #to_h (Struct, Data).
+    # A row that fails to encode leaves nothing behind in the buffers. Flushes a row group when the
+    # buffered rows reach the size limits.
+    #
+    # @param row [Hash, Array, #attributes, #to_h] row to append
+    # @return [self]
+    # @raise [Error] when the writer is closed
+    # @raise [EncodeError] when a required field is nil or missing, or a value cannot be encoded
     def <<(row)
       raise Error, "Writer is closed" if @closed
       row = row_hash(row)
@@ -123,7 +203,7 @@ module Herringbone
       rescue EncodeError => e
         rollback_row(marks)
         raise EncodeError, "Row #{@total_rows + @buffered_rows}: #{e.message}"
-      rescue StandardError
+      rescue
         rollback_row(marks)
         raise
       end
@@ -131,17 +211,15 @@ module Herringbone
       check_row_group_size if @buffered_rows >= @row_limit
       self
     end
-    alias_method :write, :<<
-
-    def write_rows(rows)
-      rows.each { |row| self << row }
-      self
-    end
 
     # Number of rows written so far, including buffered ones
+    #
+    # @return [Integer]
     def rows_written = @total_rows + @buffered_rows
 
     # Writes any buffered rows as a row group
+    #
+    # @return [void]
     def flush_row_group
       return if @buffered_rows.zero?
       start = @pos
@@ -154,20 +232,26 @@ module Herringbone
         total_compressed_size: @pos - start,
         ordinal: @row_groups.size
       )
+      write_bloom_filters
       @total_rows += @buffered_rows
       reset_buffers
     end
 
+    # Flushes buffered rows, then writes the page indexes and the footer and flushes the IO.
+    # Does nothing when already closed or aborted. The IO itself is not closed.
+    #
+    # @return [void]
     def close
       return if @closed
       flush_row_group
+      write_page_indexes
       meta = Format::FileMetaData.new(
         version: 2,
         schema: @schema.to_elements,
         num_rows: @total_rows,
         row_groups: @row_groups,
         key_value_metadata: @metadata.empty? ? nil : @metadata.map { |k, v| Format::KeyValue.new(key: k.to_s, value: v&.to_s) },
-        created_by: "herringbone version #{VERSION}",
+        created_by: "herringbone-ruby #{VERSION}",
         column_orders: @schema.columns.map { Format::ColumnOrder.new(type_order: Format::TypeDefinedOrder.new) }
       )
       footer = meta.encode
@@ -176,24 +260,51 @@ module Herringbone
       write_raw(MAGIC)
       @io.flush if @io.respond_to?(:flush)
       @closed = true
-      finish_target
     end
 
-    # Stops writing without producing a file: a temporary file is deleted, an IO is left as is
+    # Stops writing without finishing the file (no footer is written). Whatever was already written
+    # to the IO stays there; discarding it is up to the caller.
+    #
+    # @return [void]
     def abort
-      return if @closed
       @closed = true
-      return unless @temp_path
-      @io.close unless @io.closed?
-      File.unlink(@temp_path) if File.exist?(@temp_path)
     end
 
     private
 
+    # Pathname responds to #write too (writing a whole file by path), so it is rejected explicitly.
+    # A text-mode IO (a pipe, or a File opened with "w") transcodes what is written when
+    # Encoding.default_internal is set, as Rails does, and binary pages cannot be transcoded,
+    # so the IO is switched to binary mode.
+    #
+    # @param io [Object] candidate destination
+    # @return [IO, #write] +io+, in binary mode when it supports #binmode
+    # @raise [ArgumentError] when +io+ does not respond to #write, or is a String or Pathname
+    def check_io!(io)
+      if io.respond_to?(:write) && !io.is_a?(String) && !(defined?(Pathname) && io.is_a?(Pathname))
+        io.binmode if io.respond_to?(:binmode)
+        return io
+      end
+      raise ArgumentError, "Herringbone::Writer expects an IO that responds to #write " \
+        "(e.g. File.open(path, \"wb\") or StringIO.new), got #{io.class}"
+    end
+
     # Levels are kept as binary Strings (one byte per entry). Values are an Array for numeric and
     # boolean columns, and a compact ByteValues for BYTE_ARRAY / FIXED_LEN_BYTE_ARRAY columns.
+    #
+    # @!attribute defs
+    #   @return [String, Array<Integer>] definition levels (unpacked to an Array while a chunk is written)
+    # @!attribute reps
+    #   @return [String, Array<Integer>, nil] repetition levels, like +defs+; nil for non-repeated columns
+    # @!attribute values
+    #   @return [Array, ByteValues] non-null physical values
     ColumnBuffer = Struct.new(:defs, :reps, :values)
 
+    # Starts a new row group: fresh buffers per column, and the per-field write plan
+    # (field, name, Symbol name, buffer and encoder for flat fields, or nils for shredded ones).
+    #
+    # @return [void]
+    # @raise [UnsupportedError] when a column has more than 255 definition or repetition levels
     def reset_buffers
       @buffers = @schema.columns.map do |col|
         if col.max_definition_level > 255 || col.max_repetition_level > 255
@@ -212,6 +323,8 @@ module Herringbone
       @buffered_rows = 0
     end
 
+    # @param col [Schema::Column] column to buffer
+    # @return [ByteValues, Array] empty value store for the column
     def new_values_store(col)
       case col.type
       when T::BYTE_ARRAY then ByteValues.new(dictionary: use_dictionary?(col))
@@ -220,25 +333,10 @@ module Herringbone
       end
     end
 
-    def open_target(target)
-      if target.respond_to?(:write)
-        @io = target
-      else
-        @path = target.to_s
-        dir = File.dirname(@path)
-        @temp_path = File.join(dir, ".#{File.basename(@path)}.#{Process.pid}.#{rand(1 << 32).to_s(36)}.tmp")
-        @io = File.open(@temp_path, "wb")
-      end
-    end
-
-    def finish_target
-      return unless @temp_path
-      @io.close
-      File.rename(@temp_path, @path)
-    end
-
-    # Flushes when the buffered values reach row_group_bytes (or row_group_size rows). The bytes per
+    # Flushes when the buffered values reach row_group_bytes (or row_group_rows rows). The bytes per
     # row are estimated from the buffered values after the first rows, then refreshed per row group.
+    #
+    # @return [void]
     def check_row_group_size
       @bytes_per_row ||= estimate_bytes_per_row
       limit = row_limit_for(@bytes_per_row)
@@ -251,11 +349,15 @@ module Herringbone
       end
     end
 
+    # @param bytes_per_row [Integer] estimated buffered bytes per row
+    # @return [Integer] rows to buffer before the next size check: what fits in row_group_bytes,
+    #   at least ESTIMATE_AFTER_ROWS, at most row_group_rows
     def row_limit_for(bytes_per_row)
       by_bytes = [@row_group_bytes / [bytes_per_row, 1].max, ESTIMATE_AFTER_ROWS].max
-      @row_group_size ? [@row_group_size, by_bytes].min : by_bytes
+      @row_group_rows ? [@row_group_rows, by_bytes].min : by_bytes
     end
 
+    # @return [Integer] memory held by the buffers divided by the buffered rows
     def estimate_bytes_per_row
       bytes = @schema.columns.sum do |col|
         buf = @buffers[col.index]
@@ -263,29 +365,58 @@ module Herringbone
         held = if values.is_a?(ByteValues)
           values.memory_bytes
         else
-          values.size * (col.type == T::INT96 ? 48 : 8)
+          values.size * ((col.type == T::INT96) ? 48 : 8)
         end
         held + buf.defs.bytesize + (buf.reps ? buf.reps.bytesize : 0)
       end
       bytes / [@buffered_rows, 1].max
     end
 
+    # Writes to the IO and tracks the file offset, since the IO is never asked for its position
+    #
+    # @param bytes [String] binary data
+    # @return [void]
     def write_raw(bytes)
       @io.write(bytes)
       @pos += bytes.bytesize
     end
 
+    # @param path [String, Symbol] column path, for the error message
+    # @param enc [Symbol, String, Integer] encoding name from ENCODING_NAMES, or an encoding id
+    # @return [Integer] encoding id
+    # @raise [ArgumentError] for an unknown encoding name, or an id that is not in VALID_ENCODINGS
     def encoding_id(path, enc)
-      return enc if enc.is_a?(Integer)
+      if enc.is_a?(Integer)
+        return enc if VALID_ENCODINGS.key?(enc)
+        raise ArgumentError, "Encoding #{E::NAMES.fetch(enc, enc)} cannot be requested for #{path}"
+      end
       ENCODING_NAMES.fetch(enc.to_s.downcase.to_sym) { raise ArgumentError, "Unknown encoding #{enc.inspect} for #{path}" }
     end
 
+    # @param col [Schema::Column] column the encoding is requested for
+    # @param enc [Integer] encoding id, a key of VALID_ENCODINGS
+    # @return [void]
+    # @raise [ArgumentError] when the encoding is not valid for the column's physical type
+    def check_encoding!(col, enc)
+      return if VALID_ENCODINGS.fetch(enc).include?(col.type)
+      raise ArgumentError, "Encoding #{E::NAMES[enc]} is not valid for #{T::NAMES[col.type]} column #{col.dotted_path}"
+    end
+
+    # Reads a struct member, by String or Symbol key
+    #
+    # @param hash [Hash, #to_h, nil] struct value
+    # @param name [String] member name
+    # @return [Object, nil] member value, nil when +hash+ is nil or has no such key
+    # @raise [EncodeError] when +hash+ cannot be converted to a Hash
     def lookup(hash, name)
       return nil if hash.nil?
       hash = as_hash(hash, name) unless hash.is_a?(Hash)
       hash.fetch(name) { hash[name.to_sym] }
     end
 
+    # @param row [Hash, Array, #attributes, #to_h] row given to #<<
+    # @return [Hash] the row keyed by top-level field names
+    # @raise [EncodeError] when an Array has the wrong size or the row cannot be converted to a Hash
     def row_hash(row)
       case row
       when Hash then row
@@ -300,6 +431,10 @@ module Herringbone
       end
     end
 
+    # @param value [Hash, #to_h] value to convert
+    # @param what [String] description of the value, for the error message
+    # @return [Hash]
+    # @raise [EncodeError] when +value+ does not respond to #to_h
     def as_hash(value, what)
       return value if value.is_a?(Hash)
       raise EncodeError, "Expected a Hash for #{what}, got #{value.class}" unless value.respond_to?(:to_h)
@@ -307,6 +442,10 @@ module Herringbone
     end
 
     # Removes the entries a failed row left behind, so the buffers stay aligned
+    #
+    # @param marks [Array<Array(Integer, Integer, Integer)>, nil] sizes of defs, reps and values of
+    #   each nested buffer before the row, nil when there are none
+    # @return [void]
     def rollback_row(marks)
       @plan.each do |field, _, _, buf|
         next unless buf && buf.defs.bytesize > @buffered_rows
@@ -323,6 +462,14 @@ module Herringbone
     end
 
     # Record shredding: turns a nested value into (definition level, repetition level, value) entries
+    #
+    # @param field [Schema::Field] field the value belongs to
+    # @param value [Object, nil] Ruby value of the field
+    # @param parent_def [Integer] definition level recorded when +value+ is nil
+    # @param rep [Integer] repetition level of the first entry this value produces
+    # @return [void]
+    # @raise [EncodeError] when a required value is nil, a list or map value has the wrong type, a
+    #   map key is nil, or a leaf value cannot be encoded
     def shred(field, value, parent_def, rep)
       if value.nil?
         raise EncodeError, "Field #{field.node.path.join(".")} is required but got nil" unless field.optional
@@ -379,6 +526,13 @@ module Herringbone
       end
     end
 
+    # Writes one column chunk of the current row group: an optional dictionary page, then data pages.
+    # Bloom filters and page indexes are queued, to be written after the row group and before the
+    # footer.
+    #
+    # @param col [Schema::Column] column being written
+    # @param buffer [ColumnBuffer] the column's buffered levels and values
+    # @return [Format::ColumnChunk] chunk with its ColumnMetaData, for the row group
     def write_column_chunk(col, buffer)
       type = col.type
       path = col.dotted_path
@@ -399,11 +553,7 @@ module Herringbone
       value_encoding = if dict_values
         E::RLE_DICTIONARY
       else
-        enc = @encodings[path] || E::PLAIN
-        unless VALID_ENCODINGS.fetch(enc).include?(type)
-          raise ArgumentError, "Encoding #{E::NAMES[enc]} is not valid for #{T::NAMES[type]} column #{path}"
-        end
-        enc
+        @encodings[path] || E::PLAIN
       end
 
       chunk_start = @pos
@@ -430,6 +580,9 @@ module Herringbone
       else
         values.sum(&:bytesize) + 4 * values.size
       end
+      order = sort_key(col)
+      pages = []
+      first_row = 0
       page_ranges(buf, value_bytes).each do |from, to|
         n = to - from
         defs = buf.defs[from, n]
@@ -437,6 +590,12 @@ module Herringbone
         non_null = max_def.zero? ? n : defs.count(max_def)
         page_values = (indices || values)[value_index, non_null]
         value_index += non_null
+        page_offset = @pos
+        range = nil
+        if order && non_null.positive?
+          in_page = dict_values ? page_values.uniq.map { |i| dict_values[i] } : page_values
+          range = value_range(col, in_page, order)
+        end
         encoded = encode_values(page_values, value_encoding, type, col.type_length, dict_values&.size)
         rep_bytes = max_rep.positive? ? Encodings::RLE.encode_hybrid(reps, max_rep.bit_length) : "".b
         def_bytes = max_def.positive? ? Encodings::RLE.encode_hybrid(defs, max_def.bit_length) : "".b
@@ -446,6 +605,8 @@ module Herringbone
           num_rows = max_rep.zero? ? n : reps.count(0)
           write_data_page_v2(n, n - non_null, num_rows, rep_bytes, def_bytes, encoded, value_encoding)
         end
+        pages << PageInfo.new(page_offset, @pos - page_offset, first_row, n - non_null, non_null, range)
+        first_row += max_rep.zero? ? n : reps.count(0)
       end
 
       encodings = [E::RLE]
@@ -461,11 +622,178 @@ module Herringbone
         total_compressed_size: @pos - chunk_start,
         data_page_offset: data_offset,
         dictionary_page_offset: dictionary_offset,
-        statistics: @statistics ? statistics_for(col, buf.defs, values || indices.uniq.map { |i| dict_values[i] }) : nil
+        statistics: statistics_for(col, buf.defs, values || indices.uniq.map { |i| dict_values[i] }, order)
+      )
+      chunk = Format::ColumnChunk.new(file_offset: chunk_start, meta_data: meta)
+      if (bloom = @bloom_filters[path])
+        @pending_bloom_filters << [meta, build_bloom_filter(col, bloom, dict_values, values)]
+      end
+      @page_indexes << [chunk, column_index_for(col, pages, order), offset_index_for(pages)]
+      chunk
+    end
+
+    # What the page indexes need to know about a written data page
+    #
+    # @!attribute offset
+    #   @return [Integer] file offset of the page header
+    # @!attribute size
+    #   @return [Integer] page size in the file, header included
+    # @!attribute first_row
+    #   @return [Integer] index of the page's first row within the row group
+    # @!attribute nulls
+    #   @return [Integer] null entries in the page
+    # @!attribute non_null
+    #   @return [Integer] non-null values in the page
+    # @!attribute range
+    #   @return [Array(Object, Object), nil] [min, max] physical values, nil when there are none
+    PageInfo = Struct.new(:offset, :size, :first_row, :nulls, :non_null, :range)
+
+    # @param pages [Array<PageInfo>] pages of a column chunk
+    # @return [Format::OffsetIndex]
+    def offset_index_for(pages)
+      Format::OffsetIndex.new(page_locations: pages.map do |p|
+        Format::PageLocation.new(offset: p.offset, compressed_page_size: p.size, first_row_index: p.first_row)
+      end)
+    end
+
+    # nil when the column has no defined sort order, or a page's values have no min/max (all NaN)
+    #
+    # @param col [Schema::Column] column the pages belong to
+    # @param pages [Array<PageInfo>] pages of the column chunk
+    # @param order [Proc, Method, nil] sort key from #sort_key
+    # @return [Format::ColumnIndex, nil]
+    def column_index_for(col, pages, order)
+      return nil unless order
+      return nil if pages.any? { |p| p.non_null.positive? && p.range.nil? }
+      Format::ColumnIndex.new(
+        null_pages: pages.map { |p| p.non_null.zero? },
+        min_values: pages.map { |p| p.range ? truncate_min(stat_bytes(col, p.range[0])) : "".b },
+        max_values: pages.map { |p| p.range ? truncate_max(stat_bytes(col, p.range[1])) : "".b },
+        boundary_order: boundary_order(pages.filter_map(&:range), order),
+        null_counts: pages.map(&:nulls)
       )
-      Format::ColumnChunk.new(file_offset: chunk_start, meta_data: meta)
     end
 
+    # @param ranges [Array<Array(Object, Object)>] [min, max] of each non-null page, in page order
+    # @param order [Proc, Method] sort key from #sort_key
+    # @return [Integer] Format::BoundaryOrder: ASCENDING when both mins and maxes never decrease
+    #   (or there are fewer than two pages), DESCENDING when they never increase, else UNORDERED
+    def boundary_order(ranges, order)
+      return Format::BoundaryOrder::ASCENDING if ranges.size < 2
+      cmp = ->(a, b) { order.equal?(IDENTITY) ? a <=> b : order.call(a) <=> order.call(b) }
+      pairs = ranges.each_cons(2)
+      if pairs.all? { |(a_min, a_max), (b_min, b_max)| cmp.call(a_min, b_min) <= 0 && cmp.call(a_max, b_max) <= 0 }
+        Format::BoundaryOrder::ASCENDING
+      elsif pairs.all? { |(a_min, a_max), (b_min, b_max)| cmp.call(a_min, b_min) >= 0 && cmp.call(a_max, b_max) >= 0 }
+        Format::BoundaryOrder::DESCENDING
+      else
+        Format::BoundaryOrder::UNORDERED
+      end
+    end
+
+    # Page indexes go after the last row group: all column indexes, then all offset indexes
+    #
+    # @return [void]
+    def write_page_indexes
+      @page_indexes.each do |chunk, column_index, _|
+        next unless column_index
+        bytes = column_index.encode
+        chunk.column_index_offset = @pos
+        chunk.column_index_length = bytes.bytesize
+        write_raw(bytes)
+      end
+      @page_indexes.each do |chunk, _, offset_index|
+        bytes = offset_index.encode
+        chunk.offset_index_offset = @pos
+        chunk.offset_index_length = bytes.bytesize
+        write_raw(bytes)
+      end
+      @page_indexes.clear
+    end
+
+    # Per-column settings accepted in the +bloom_filters:+ option
+    BLOOM_FILTER_OPTIONS = %i[ndv fpp max_bytes].freeze
+
+    # { dotted_path => { ndv:, fpp:, max_bytes: } } from the bloom_filters: option
+    #
+    # @param requested [Boolean, Array<String>, Hash{String => Boolean, Hash, nil}, nil] true for every
+    #   column of a supported type, column paths, or column path => true / false / settings Hash.
+    #   A path may also be given as an Array of names. Settings are +ndv+, +fpp+ and +max_bytes+.
+    # @return [Hash{String => Hash{Symbol => Numeric, nil}}] settings by dotted path; empty when disabled
+    # @raise [ArgumentError] for an unknown column, a column type without bloom filter support, or
+    #   invalid settings
+    def bloom_filter_config(requested)
+      return {} if requested.nil? || requested == false
+      columns = @schema.columns.to_h { |c| [c.dotted_path, c] }
+      if requested == true
+        return columns.select { |_, c| BloomFilter::TYPES.include?(c.type) }.transform_values { bloom_filter_settings({}) }
+      end
+      requested = Array(requested).to_h { |path| [path, true] } unless requested.is_a?(Hash)
+      requested.each_with_object({}) do |(path, settings), config|
+        path = path.is_a?(Array) ? path.join(".") : path.to_s
+        col = columns[path] or raise ArgumentError, "bloom_filters: no such column #{path}"
+        next if settings.nil? || settings == false
+        unless BloomFilter::TYPES.include?(col.type)
+          raise ArgumentError, "bloom_filters: not supported for #{T::NAMES[col.type]} column #{path}"
+        end
+        settings = {} if settings == true
+        raise ArgumentError, "bloom_filters: expected true or a Hash for #{path}, got #{settings.inspect}" unless settings.is_a?(Hash)
+        config[path] = bloom_filter_settings(settings.transform_keys(&:to_sym), path)
+      end
+    end
+
+    # @param settings [Hash{Symbol => Object}] +ndv+, +fpp+ and +max_bytes+, all optional
+    # @param path [String, nil] column path, for error messages
+    # @return [Hash{Symbol => Numeric, nil}] +ndv+ (nil to count distinct values), +fpp+ and
+    #   +max_bytes+ with defaults filled in
+    # @raise [ArgumentError] for unknown keys, a non-positive ndv, or an fpp outside (0, 1)
+    def bloom_filter_settings(settings, path = nil)
+      unknown = settings.keys - BLOOM_FILTER_OPTIONS
+      raise ArgumentError, "bloom_filters: unknown option #{unknown.join(", ")} for #{path}" unless unknown.empty?
+      ndv = settings[:ndv] && Integer(settings[:ndv])
+      raise ArgumentError, "bloom_filters: ndv must be positive for #{path}" if ndv && !ndv.positive?
+      fpp = Float(settings.fetch(:fpp, BloomFilter::DEFAULT_FPP))
+      raise ArgumentError, "bloom_filters: fpp must be between 0 and 1 for #{path}" unless fpp > 0 && fpp < 1
+      {ndv: ndv, fpp: fpp, max_bytes: Integer(settings.fetch(:max_bytes, BloomFilter::DEFAULT_MAX_BYTES))}
+    end
+
+    # A filter holding the chunk's values: +dict_values+ (already distinct) for dictionary-encoded
+    # chunks, +values+ otherwise. Each distinct value is hashed once, and the filter is sized from
+    # the configured ndv or from the number of distinct values.
+    #
+    # @param col [Schema::Column] column the filter is for
+    # @param settings [Hash{Symbol => Numeric, nil}] from #bloom_filter_settings
+    # @param dict_values [Array, nil] dictionary of the chunk, when dictionary-encoded
+    # @param values [Array, nil] the chunk's non-null physical values, used without a dictionary
+    # @return [BloomFilter]
+    def build_bloom_filter(col, settings, dict_values, values)
+      hashes = if dict_values
+        BloomFilter.hash_physical_all(dict_values, col.type)
+      else
+        BloomFilter.hash_physical_all(values, col.type, distinct: true)
+      end
+      size = BloomFilter.optimal_num_bytes(settings[:ndv] || hashes.size, settings[:fpp], max_bytes: settings[:max_bytes])
+      filter = BloomFilter.new(size, column: col)
+      filter.insert_hashes(hashes)
+      filter
+    end
+
+    # Bloom filters go right after the row group's column chunks, in column order
+    #
+    # @return [void]
+    def write_bloom_filters
+      @pending_bloom_filters.each do |meta, filter|
+        bytes = filter.encode
+        meta.bloom_filter_offset = @pos
+        meta.bloom_filter_length = bytes.bytesize
+        write_raw(bytes)
+      end
+      @pending_bloom_filters.clear
+    end
+
+    # @param col [Schema::Column] column to check
+    # @return [Boolean] whether the +dictionary:+ option asks for this column to be dictionary-encoded
+    #   (never for BOOLEAN)
     def use_dictionary?(col)
       return false if col.type == T::BOOLEAN
       case @dictionary
@@ -475,13 +803,20 @@ module Herringbone
       end
     end
 
-    # Returns [dictionary_values, indices] or nil when a dictionary is not worthwhile
+    # Returns [dictionary_values, indices] or nil when a dictionary is not worthwhile: more than about
+    # half the values are distinct, or the dictionary exceeds MAX_DICTIONARY_BYTES
+    #
+    # @param values [Array] non-null physical values of the chunk
+    # @param type [Integer] physical type
+    # @param type_length [Integer, nil] FIXED_LEN_BYTE_ARRAY width
+    # @return [Array(Array, Array<Integer>), nil]
     def build_dictionary(values, type, type_length)
       # Floats are keyed by bit pattern so that -0.0 and 0.0 (and NaNs) stay distinct
       if type == T::FLOAT || type == T::DOUBLE
         keys = values.pack("G*").unpack("Q>*")
         uniq = keys.uniq
         return nil if uniq.size > values.size / 2 + 1 && values.size > 16
+        return nil if uniq.size * 8 > MAX_DICTIONARY_BYTES
         index = uniq.each_with_index.to_h
         return [uniq.pack("Q>*").unpack("G*"), keys.map(&index)]
       end
@@ -497,14 +832,19 @@ module Herringbone
       [dict, values.map(&index)]
     end
 
-    # Splits a column buffer into pages of roughly @page_size bytes. Repeated columns
+    # Splits a column buffer into pages of roughly @page_bytes bytes. Repeated columns
     # are only cut where a new row starts.
+    #
+    # @param buf [ColumnBuffer] column buffer with levels as Arrays
+    # @param value_bytes [Integer] estimated encoded size of all the chunk's values
+    # @return [Array<Array(Integer, Integer)>] [from, to) ranges of level entries, one per page
     def page_ranges(buf, value_bytes)
       n = buf.defs.size
       bytes = n + value_bytes
-      pages = (bytes + @page_size - 1) / @page_size
-      return [[0, n]] if pages <= 1
-      per = (n + pages - 1) / pages
+      pages = (bytes + @page_bytes - 1) / @page_bytes
+      per = (pages <= 1) ? n : (n + pages - 1) / pages
+      per = @page_rows if per > @page_rows
+      return [[0, n]] if per >= n
       reps = buf.reps
       ranges = []
       start = 0
@@ -518,6 +858,8 @@ module Herringbone
       ranges
     end
 
+    # @param col [Schema::Column] column to check
+    # @return [Integer, nil] PLAIN-encoded bytes per value, nil for BYTE_ARRAY (variable width)
     def value_width(col)
       case col.type
       when T::BOOLEAN then 1
@@ -528,6 +870,15 @@ module Herringbone
       end
     end
 
+    # Encodes a page's values. RLE_DICTIONARY values are the dictionary indices, prefixed with the
+    # bit width byte; RLE is only used for BOOLEAN and carries the 4-byte length prefix.
+    #
+    # @param values [Array] physical values, or dictionary indices
+    # @param encoding [Integer] encoding id
+    # @param type [Integer] physical type
+    # @param type_length [Integer, nil] FIXED_LEN_BYTE_ARRAY width
+    # @param dict_size [Integer, nil] number of dictionary entries, for RLE_DICTIONARY
+    # @return [String] encoded binary page values
     def encode_values(values, encoding, type, type_length, dict_size)
       case encoding
       when E::PLAIN then Encodings::Plain.encode(values, type, type_length)
@@ -538,27 +889,41 @@ module Herringbone
         body = Encodings::RLE.encode_hybrid(values.map { |v| v ? 1 : 0 }, 1)
         [body.bytesize].pack("V") << body
       when E::DELTA_BINARY_PACKED
-        Encodings::Delta.encode_binary_packed(values, type == T::INT32 ? 32 : 64)
+        Encodings::Delta.encode_binary_packed(values, (type == T::INT32) ? 32 : 64)
       when E::DELTA_LENGTH_BYTE_ARRAY then Encodings::Delta.encode_length_byte_array(values)
       when E::DELTA_BYTE_ARRAY then Encodings::Delta.encode_byte_array(values)
       when E::BYTE_STREAM_SPLIT
-        width = { T::INT32 => 4, T::FLOAT => 4, T::INT64 => 8, T::DOUBLE => 8 }[type] || type_length
+        width = {T::INT32 => 4, T::FLOAT => 4, T::INT64 => 8, T::DOUBLE => 8}[type] || type_length
         Encodings::ByteStreamSplit.encode(Encodings::Plain.encode(values, type, type_length), width)
       end
     end
 
     # Writes a page, returning the uncompressed size including the header
+    #
+    # @param header [Format::PageHeader] header; sizes and CRC32 are filled in here
+    # @param body [String] uncompressed page body
+    # @param compressed [String, nil] bytes to write as the page body, when already prepared (v2 data
+    #   pages, whose levels stay uncompressed); +body+ is compressed otherwise
+    # @return [Integer]
     def write_page(header, body, compressed = nil)
       compressed ||= Compression.compress(@codec, body)
       header.uncompressed_page_size ||= body.bytesize
       header.compressed_page_size = compressed.bytesize
-      header.crc = Zlib.crc32(compressed).then { |c| c >= 0x8000_0000 ? c - 0x1_0000_0000 : c }
+      header.crc = Zlib.crc32(compressed).then { |c| (c >= 0x8000_0000) ? c - 0x1_0000_0000 : c }
       encoded = header.encode
       write_raw(encoded)
       write_raw(compressed)
       encoded.bytesize + header.uncompressed_page_size
     end
 
+    # A DATA_PAGE: length-prefixed repetition and definition levels, then the values, all compressed
+    #
+    # @param n [Integer] number of level entries (values including nulls)
+    # @param rep_bytes [String] RLE-encoded repetition levels, empty when the column has none
+    # @param def_bytes [String] RLE-encoded definition levels, empty when the column has none
+    # @param encoded [String] encoded values
+    # @param encoding [Integer] encoding id of the values
+    # @return [Integer] uncompressed size including the header
     def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding)
       body = String.new(encoding: Encoding::BINARY)
       body << [rep_bytes.bytesize].pack("V") << rep_bytes unless rep_bytes.empty?
@@ -574,6 +939,16 @@ module Herringbone
       write_page(header, body)
     end
 
+    # A DATA_PAGE_V2: levels without length prefixes and uncompressed, then the compressed values
+    #
+    # @param n [Integer] number of level entries (values including nulls)
+    # @param nulls [Integer] null entries
+    # @param rows [Integer] rows in the page
+    # @param rep_bytes [String] RLE-encoded repetition levels, empty when the column has none
+    # @param def_bytes [String] RLE-encoded definition levels, empty when the column has none
+    # @param encoded [String] encoded values
+    # @param encoding [Integer] encoding id of the values
+    # @return [Integer] uncompressed size including the header
     def write_data_page_v2(n, nulls, rows, rep_bytes, def_bytes, encoded, encoding)
       compressed = Compression.compress(@codec, encoded)
       header = Format::PageHeader.new(
@@ -589,46 +964,116 @@ module Herringbone
       write_page(header, "".b, rep_bytes + def_bytes + compressed)
     end
 
-    def statistics_for(col, defs, values)
+    # Statistics and column index bounds longer than this are truncated, see #truncate_min
+    STAT_TRUNCATE_BYTES = 64
+    # Sort key for types whose Ruby ordering already matches Parquet's; compared by identity to
+    # skip calling it
+    IDENTITY = ->(v) { v }
+
+    # Chunk statistics: null count, plus min/max (flagged exact unless truncated) when the column
+    # has a sort order and non-NaN values
+    #
+    # @param col [Schema::Column] column the chunk belongs to
+    # @param defs [Array<Integer>] definition levels of the chunk
+    # @param values [Array] distinct or all non-null physical values of the chunk
+    # @param order [Proc, Method, nil] sort key from #sort_key
+    # @return [Format::Statistics]
+    def statistics_for(col, defs, values, order)
       max_def = col.max_definition_level
       nulls = max_def.zero? ? 0 : defs.size - defs.count(max_def)
       stats = Format::Statistics.new(null_count: nulls)
-      min, max = min_max(col, values)
-      if min
-        stats.min_value = min
-        stats.max_value = max
-        stats.is_min_value_exact = true
-        stats.is_max_value_exact = true
+      range = order && value_range(col, values, order)
+      if range
+        min = stat_bytes(col, range[0])
+        max = stat_bytes(col, range[1])
+        stats.min_value = truncate_min(min)
+        stats.max_value = truncate_max(max)
+        stats.is_min_value_exact = stats.min_value.bytesize == min.bytesize
+        stats.is_max_value_exact = stats.max_value.bytesize == max.bytesize
       end
       stats
     end
 
-    # Min/max for types whose Parquet sort order matches Ruby's comparison of the physical values
-    def min_max(col, values)
-      return nil if values.empty?
+    # A key giving the Parquet sort order of the column's physical values (IDENTITY when Ruby's own
+    # comparison already matches), or nil when the order is undefined (INT96)
+    #
+    # @param col [Schema::Column] column to order
+    # @return [Proc, Method, nil]
+    def sort_key(col)
       kind, _, signed = Types.logical_of(col.node)
       case col.type
-      when T::BOOLEAN
-        [values.include?(false) ? "\x00".b : "\x01".b, values.include?(true) ? "\x01".b : "\x00".b]
+      when T::BOOLEAN then ->(v) { v ? 1 : 0 }
       when T::INT32, T::INT64
-        return nil if kind == :integer && !signed
-        fmt = col.type == T::INT32 ? "l<" : "q<"
-        min, max = values.minmax
-        [[min].pack(fmt), [max].pack(fmt)]
-      when T::FLOAT, T::DOUBLE
-        finite = values.reject(&:nan?)
-        return nil if finite.empty?
-        min, max = finite.minmax
+        return IDENTITY unless kind == :integer && !signed
+        mask = (col.type == T::INT32) ? 0xFFFF_FFFF : 0xFFFF_FFFF_FFFF_FFFF
+        ->(v) { v & mask }
+      when T::FLOAT, T::DOUBLE then IDENTITY
+      when T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY
+        case kind
+        when :decimal then Types.method(:be_to_int)
+        when :float16 then ->(v) { Types.half_to_float(v.unpack1("v")) }
+        else IDENTITY # unsigned lexicographic, which is how Ruby compares binary Strings
+        end
+      end
+    end
+
+    # [min, max] of +values+ in column order, ignoring NaNs; nil if there is nothing to compare.
+    # A zero float bound is normalized to -0.0 (min) / 0.0 (max), as the spec asks.
+    #
+    # @param col [Schema::Column] column the values belong to
+    # @param values [Array] physical values
+    # @param order [Proc, Method] sort key from #sort_key
+    # @return [Array(Object, Object), nil]
+    def value_range(col, values, order)
+      floats = col.type == T::FLOAT || col.type == T::DOUBLE
+      if floats
+        values = values.reject(&:nan?)
+      elsif Types.logical_of(col.node).first == :float16
+        values = values.reject { |v| order.call(v).nan? }
+      end
+      return nil if values.empty?
+      min, max = order.equal?(IDENTITY) ? values.minmax : values.minmax_by(&order)
+      if floats
         min = -0.0 if min.zero?
         max = 0.0 if max.zero?
-        fmt = col.type == T::FLOAT ? "e" : "E"
-        [[min].pack(fmt), [max].pack(fmt)]
-      when T::BYTE_ARRAY
-        return nil if kind == :decimal
-        min, max = values.minmax
-        return nil if min.bytesize > 1024 || max.bytesize > 1024
-        [min, max]
       end
+      [min, max]
+    end
+
+    # @param col [Schema::Column] column the value belongs to
+    # @param value [Object] physical value
+    # @return [String] the value PLAIN-encoded, as statistics store it (byte arrays without length)
+    def stat_bytes(col, value)
+      case col.type
+      when T::BOOLEAN then value ? "\x01".b : "\x00".b
+      when T::INT32 then [value].pack("l<")
+      when T::INT64 then [value].pack("q<")
+      when T::FLOAT then [value].pack("e")
+      when T::DOUBLE then [value].pack("E")
+      else value.b
+      end
+    end
+
+    # Long byte-array bounds are truncated: a prefix is still a lower bound for the minimum, and
+    # a prefix with its last byte incremented is an upper bound for the maximum
+    #
+    # @param bytes [String] encoded minimum
+    # @return [String] at most STAT_TRUNCATE_BYTES bytes
+    def truncate_min(bytes)
+      (bytes.bytesize > STAT_TRUNCATE_BYTES) ? bytes.byteslice(0, STAT_TRUNCATE_BYTES) : bytes
+    end
+
+    # @param bytes [String] encoded maximum
+    # @return [String] at most STAT_TRUNCATE_BYTES bytes, with the last byte that is not 0xFF
+    #   incremented (trailing 0xFF bytes dropped); +bytes+ unchanged when it fits, or when the
+    #   whole prefix is 0xFF
+    def truncate_max(bytes)
+      return bytes if bytes.bytesize <= STAT_TRUNCATE_BYTES
+      prefix = bytes.byteslice(0, STAT_TRUNCATE_BYTES).bytes
+      prefix.pop while prefix.last == 0xFF
+      return bytes if prefix.empty?
+      prefix[-1] += 1
+      prefix.pack("C*")
     end
   end
 end