herringbone 0.3.0 → 0.5.0

data/lib/herringbone/simple_writer.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Herringbone
+  # Writes Parquet the way the CSV gem writes CSV: name the columns, then append rows as Arrays.
+  # The column types are inferred from the first Schema::INFER_SAMPLE rows (see Schema.infer), so
+  # no schema has to be declared.
+  #
+  #   File.open("people.parquet", "wb") do |file|
+  #     Herringbone::SimpleWriter.open(file) do |sw|
+  #       sw.headers!(:id, :name, :age)
+  #       sw << [123, "John", 12]
+  #       sw << { id: 124, name: "Jane" } # Hashes work too; missing columns are nulls
+  #     end
+  #   end
+  #
+  # Every column is nullable. A column that is nil in every sampled row becomes a string, and a file
+  # with headers but no rows has only string columns. Only the sample is held in memory; later rows
+  # are written as they come. A row that does not fit the inferred types raises SchemaMismatch,
+  # explaining what was inferred and how to declare the column, and the writer stops there (the
+  # file is left without a footer). Unlike CSV, the headers are mandatory: a Parquet file always
+  # has named columns.
+  #
+  # Columns declared in a block (Schema::Builder DSL) replace inferred ones, e.g. for a column that
+  # holds both numbers and text:
+  #
+  #   Herringbone::SimpleWriter.new(io) { |s| s.string :code }
+  class SimpleWriter
+    # Opens a writer, yields it and closes it, finishing the file. If the block raises, the file is
+    # left unfinished (no footer), as with Writer.open. To declare columns, use #initialize and #close.
+    #
+    # @param io [IO, #write] destination; written sequentially, never closed
+    # @param options [Hash{Symbol => Object}] Writer options (compression:, row_group_bytes:...)
+    # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
+    # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
+    #   (other Writer options are passed on as well)
+    # @yield [writer] the open writer
+    # @yieldparam writer [SimpleWriter]
+    # @yieldreturn [Object] returned by open
+    # @return [SimpleWriter, Object] the writer without a block, the block's value with one
+    def self.open(io, **options)
+      writer = new(io, **options)
+      return writer unless block_given?
+      begin
+        result = yield writer
+      rescue Exception # rubocop:disable Lint/RescueException -- also abort on Interrupt
+        writer.abort
+        raise
+      end
+      writer.close
+      result
+    end
+
+    # @return [Array<String>, nil] the column names, nil until #headers! is called
+    attr_reader :headers
+
+    # @param io [IO, #write] destination; nothing is written to it until the column types are known
+    # @param options [Hash{Symbol => Object}] Writer options (compression:, row_group_bytes:...)
+    # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
+    # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
+    #   (other Writer options are passed on as well)
+    # @yield [s] optional, declares columns (named in #headers!) that replace inferred ones
+    # @yieldparam s [Schema::Builder] the builder to declare columns on
+    # @yieldreturn [void]
+    # @raise [MissingCodecError] when the codec's optional gem is not loaded
+    # @raise [ArgumentError] when the block takes no parameter
+    def initialize(io, **options, &overrides)
+      Schema::Builder.check_block!(overrides, "Herringbone::SimpleWriter.new(io) { |s| s.string :code }")
+      @headers = nil
+      @overrides = overrides
+      fix = "Herringbone::SimpleWriter.new(io) { |s| s.%s }"
+      @writer = InferringWriter.new(io, fix: fix, **options) { |sample| schema_for(sample) }
+    end
+
+    # Names the columns, in the order row Arrays list their values. Must be called once, before
+    # the first row.
+    #
+    # @param names [Array<String, Symbol>] column names
+    # @return [self]
+    # @raise [ArgumentError] when called twice, with no names, or with a name given twice
+    def headers!(*names)
+      raise ArgumentError, "headers! was already called" if @headers
+      names = names.flatten.map(&:to_s)
+      raise ArgumentError, "headers! needs at least one column name" if names.empty?
+      duplicates = names.tally.select { |_, n| n > 1 }.keys
+      raise ArgumentError, "Duplicate column names: #{duplicates.join(", ")}" unless duplicates.empty?
+      @headers = names.freeze
+      self
+    end
+
+    # Appends a row: an Array with one value per header, in header order, or a Hash keyed by
+    # header (String or Symbol keys; missing columns are nulls).
+    #
+    # @param row [Array, Hash]
+    # @return [self]
+    # @raise [ArgumentError] when #headers! was not called, an Array has the wrong length or a Hash
+    #   has keys that are not headers
+    # @raise [SchemaMismatch] when a value does not fit its column's inferred type
+    def <<(row)
+      raise ArgumentError, "Call headers! with the column names before writing rows" unless @headers
+      @writer << values_of(row)
+      self
+    end
+
+    # @return [Integer] rows written so far, including those held back to infer the column types
+    def rows_written = @writer.rows_written
+
+    # Writes any rows still held back and finishes the file. The IO is not closed.
+    #
+    # @return [void]
+    # @raise [ArgumentError] when #headers! was never called
+    def close
+      raise ArgumentError, "Call headers! with the column names before closing" unless @headers
+      @writer.close
+    end
+
+    # Stops without finishing the file. What was already written stays in the IO.
+    #
+    # @return [void]
+    def abort = @writer.abort
+
+    # Short summary for the console, without the rows
+    #
+    # @return [String] the number of columns and the underlying writer's summary
+    def inspect
+      "#<#{self.class.name} columns=#{@headers&.size.inspect} writer=#{@writer.inspect}>"
+    end
+
+    private
+
+    # @param row [Array, Hash] row given to #<<
+    # @return [Array] its values in header order
+    # @raise [ArgumentError] when it does not match the headers
+    def values_of(row)
+      case row
+      when Array
+        unless row.size == @headers.size
+          raise ArgumentError, "Expected #{@headers.size} values (#{@headers.join(", ")}), got #{row.size}"
+        end
+        row
+      when Hash
+        unknown = row.keys.map(&:to_s) - @headers
+        raise ArgumentError, "Not among the headers (#{@headers.join(", ")}): #{unknown.join(", ")}" unless unknown.empty?
+        @headers.map { |name| row.fetch(name) { row[name.to_sym] } }
+      else
+        raise ArgumentError, "Rows must be Arrays or Hashes, got #{row.class}"
+      end
+    end
+
+    # @param sample [Array<Array>] held-back rows, values in header order
+    # @return [Schema] inferred from the sample; string columns when it is empty
+    # @raise [ArgumentError] when a column cannot be inferred or a declared column is not a header
+    def schema_for(sample)
+      rows = sample.empty? ? [@headers.to_h { |name| [name, nil] }] : sample.map { |values| @headers.zip(values).to_h }
+      schema = Schema.infer(rows, &@overrides)
+      extra = schema.fields.map(&:name) - @headers
+      raise ArgumentError, "Declared columns not among the headers: #{extra.join(", ")}" unless extra.empty?
+      schema
+    end
+  end
+end

data/lib/herringbone/thrift.rb

@@ -401,12 +401,12 @@ module Herringbone
       class << self
         # @return [Array<Field>] declared fields, sorted by id
         def fields
-          @fields ||= []
+          @fields || []
         end
 
         # @return [Hash{Integer => Field}] declared fields by id, for decoding
         def fields_by_id
-          @fields_by_id ||= fields.to_h { |f| [f.id, f] }
+          @fields_by_id || {}
         end
 
         # Declares a field and defines its accessor
@@ -417,9 +417,9 @@ module Herringbone
         #   or a Struct subclass
         # @return [void]
         def field(id, name, type)
-          fields << Field.new(id, name, type, :"@#{name}")
-          fields.sort_by!(&:id)
-          @fields_by_id = nil
+          # Shareable, so that structs can be decoded in any Ractor
+          @fields = Ractor.make_shareable((fields + [Field.new(id, name, type, :"@#{name}")]).sort_by(&:id))
+          @fields_by_id = Ractor.make_shareable(@fields.to_h { |f| [f.id, f] })
           attr_accessor name
         end
 

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/herringbone/writer.rb

@@ -3,10 +3,10 @@
 module Herringbone
   # Writes Parquet files.
   #
-  #   schema = Herringbone::Schema.define do
-  #     int64 :id, null: false
-  #     string :name
-  #     list :tags, :string
+  #   schema = Herringbone::Schema.define do |s|
+  #     s.int64 :id, null: false
+  #     s.string :name
+  #     s.list :tags, :string
   #   end
   #   File.open("out.parquet", "wb") do |file|
   #     Herringbone::Writer.open(file, schema) do |w|
@@ -56,14 +56,14 @@ module Herringbone
     }.freeze
 
     # Encoding id => physical types it may be used for, per the Parquet encodings spec
-    VALID_ENCODINGS = {
+    VALID_ENCODINGS = Ractor.make_shareable({
       E::PLAIN => T::NAMES.keys,
       E::RLE => [T::BOOLEAN],
       E::DELTA_BINARY_PACKED => [T::INT32, T::INT64],
       E::DELTA_LENGTH_BYTE_ARRAY => [T::BYTE_ARRAY],
       E::DELTA_BYTE_ARRAY => [T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY],
       E::BYTE_STREAM_SPLIT => [T::INT32, T::INT64, T::FLOAT, T::DOUBLE, T::FIXED_LEN_BYTE_ARRAY]
-    }.freeze
+    })
 
     # Largest dictionary build_dictionary keeps; above it the chunk is written without a dictionary.
     # Byte-array columns are dictionary-encoded by ByteValues, which applies its own
@@ -132,7 +132,7 @@ module Herringbone
     # @param bloom_filters [Boolean, Array<String>, Hash{String => Boolean, Hash}, nil] see the class
     #   description
     # @raise [ArgumentError] for an invalid IO, schema or option
-    # @raise [MissingCodecError] when the codec's optional gem cannot be loaded
+    # @raise [MissingCodecError] when the codec's optional gem is not loaded
     # @raise [UnsupportedError] when the codec is not supported
     def initialize(io, schema, compression: :snappy, row_group_bytes: 16 * 1024 * 1024, row_group_rows: nil,
       page_bytes: 1024 * 1024, page_rows: 20_000, data_page_version: 1, dictionary: true, encodings: {},
@@ -164,6 +164,7 @@ module Herringbone
       @total_rows = 0
       @pos = 0
       @closed = false
+      @aborted = false
       @bytes_per_row = nil
       @io = check_io!(io)
       write_raw(MAGIC)
@@ -189,20 +190,20 @@ module Herringbone
           if buf.nil?
             shred(field, value, 0, 0)
           elsif value.nil?
-            raise EncodeError, "Field #{name} is required but got nil" unless field.optional
+            raise EncodeError.new("Field #{name} is required but got nil", column: name) unless field.optional
             buf.defs << 0
           else
             begin
               buf.values << encoder.call(value)
             rescue ArgumentError, TypeError, NoMethodError, RangeError => e
-              raise EncodeError, "Cannot write #{value.inspect} to #{name}: #{e.message}"
+              raise EncodeError.new("Cannot write #{value.inspect} to #{name}: #{e.message}", column: name, value: value)
             end
             buf.defs << field.def_level
           end
         end
       rescue EncodeError => e
         rollback_row(marks)
-        raise EncodeError, "Row #{@total_rows + @buffered_rows}: #{e.message}"
+        raise EncodeError.new("Row #{@total_rows + @buffered_rows}: #{e.message}", row: @total_rows + @buffered_rows, column: e.column, value: e.value)
       rescue
         rollback_row(marks)
         raise
@@ -222,21 +223,82 @@ module Herringbone
     # @return [void]
     def flush_row_group
       return if @buffered_rows.zero?
+      write_row_group(@buffered_rows)
+    end
+
+    # A column chunk taken as it is from another file, for #write_row_group
+    #
+    # @!attribute chunk
+    #   @return [Format::ColumnChunk] the chunk's footer entry in the source file
+    # @!attribute start
+    #   @return [Integer] source file offset of the chunk's first page
+    # @!attribute bytes
+    #   @return [String] the chunk's pages, headers included
+    # @!attribute column_index
+    #   @return [String, nil] the chunk's encoded ColumnIndex
+    # @!attribute offset_index
+    #   @return [Format::OffsetIndex, nil] the chunk's OffsetIndex, with source file offsets
+    # @!attribute bloom_filter
+    #   @return [String, nil] the chunk's encoded bloom filter, header included
+    CopiedChunk = Struct.new(:chunk, :start, :bytes, :column_index, :offset_index, :bloom_filter)
+
+    # Internal (used by Redaction): writes a row group of +num_rows+ rows from the buffered values,
+    # except for the columns in +copies+, whose chunks are copied byte for byte from another file
+    # with their offsets rebased. Starts a new row group afterwards.
+    #
+    # @param num_rows [Integer] rows in the row group
+    # @param copies [Hash{Integer => CopiedChunk}] column index => chunk to copy instead of encoding
+    # @param codecs [Hash{Integer => Integer}] column index => codec id for an encoded chunk, instead
+    #   of the +compression:+ option
+    # @param bloom_filters [Hash{Integer => Boolean}] column index => true to give an encoded chunk a
+    #   bloom filter (default settings) when the +bloom_filters:+ option does not ask for one
+    # @param sorting_columns [Array<Format::SortingColumn>, nil] stored on the RowGroup as given
+    # @return [void]
+    # @raise [Error] when the writer is closed
+    def write_row_group(num_rows, copies: {}, codecs: {}, bloom_filters: {}, sorting_columns: nil)
+      raise Error, "Writer is closed" if @closed
       start = @pos
-      chunks = @schema.columns.map { |col| write_column_chunk(col, @buffers[col.index]) }
+      chunks = @schema.columns.map do |col|
+        if (copy = copies[col.index])
+          copy_column_chunk(copy)
+        else
+          bloom = @bloom_filters[col.dotted_path]
+          bloom ||= bloom_filter_settings({}) if bloom_filters[col.index] && BloomFilter::TYPES.include?(col.type)
+          write_column_chunk(col, @buffers[col.index], codec: codecs.fetch(col.index, @codec), bloom: bloom)
+        end
+      end
       @row_groups << Format::RowGroup.new(
         columns: chunks,
         total_byte_size: chunks.sum { |c| c.meta_data.total_uncompressed_size },
-        num_rows: @buffered_rows,
+        num_rows: num_rows,
+        sorting_columns: sorting_columns,
         file_offset: start,
         total_compressed_size: @pos - start,
         ordinal: @row_groups.size
       )
       write_bloom_filters
-      @total_rows += @buffered_rows
+      @total_rows += num_rows
       reset_buffers
     end
 
+    # Internal (used by Redaction): shreds one value per row of a top-level field into the buffers,
+    # for #write_row_group. Other fields are left as they are, so the caller decides which columns
+    # are encoded and which are copied.
+    #
+    # @param name [String] top-level field name
+    # @param values [Array] the field's value for each row
+    # @return [void]
+    # @raise [ArgumentError] when the schema has no such field
+    # @raise [EncodeError] when a value cannot be written to the field
+    def buffer_field(name, values)
+      field = @schema.field(name) or raise ArgumentError, "No such field #{name.inspect}"
+      values.each_with_index do |value, i|
+        shred(field, value, 0, 0)
+      rescue EncodeError => e
+        raise EncodeError.new("Row #{@total_rows + i}: #{e.message}", row: @total_rows + i, column: e.column, value: e.value)
+      end
+    end
+
     # Flushes buffered rows, then writes the page indexes and the footer and flushes the IO.
     # Does nothing when already closed or aborted. The IO itself is not closed.
     #
@@ -267,9 +329,23 @@ module Herringbone
     #
     # @return [void]
     def abort
+      @aborted = true unless @closed
       @closed = true
     end
 
+    # Short summary for the console, without the buffered values
+    #
+    # @return [String] state (open, closed or aborted), rows written including buffered ones,
+    #   row groups flushed and the codec
+    def inspect
+      state = if @aborted then "aborted"
+      elsif @closed then "closed"
+      else "open"
+      end
+      "#<#{self.class.name} #{state} rows_written=#{rows_written} row_groups=#{@row_groups.size} " \
+        "compression=#{Compression::NAMES.fetch(@codec, @codec).inspect}>"
+    end
+
     private
 
     # Pathname responds to #write too (writing a whole file by path), so it is rejected explicitly.
@@ -314,10 +390,12 @@ module Herringbone
           col.max_repetition_level.positive? ? String.new(encoding: Encoding::BINARY) : nil,
           new_values_store(col))
       end
+      # Column#encoder builds its Proc on every call
+      @encoders = @schema.columns.map(&:encoder)
       # Top-level, non-repeated leaves are written directly, everything else is shredded
       @plan = @schema.fields.map do |field|
         flat = field.leaf? && field.column.max_repetition_level.zero?
-        [field, field.name, field.name.to_sym, flat ? @buffers[field.column.index] : nil, flat ? field.column.encoder : nil]
+        [field, field.name, field.name.to_sym, flat ? @buffers[field.column.index] : nil, flat ? @encoders[field.column.index] : nil]
       end
       @nested_buffers = @plan.reject { |p| p[3] }.flat_map { |p| p[0].leaves.map { |c| @buffers[c.index] } }
       @buffered_rows = 0
@@ -472,7 +550,7 @@ module Herringbone
     #   map key is nil, or a leaf value cannot be encoded
     def shred(field, value, parent_def, rep)
       if value.nil?
-        raise EncodeError, "Field #{field.node.path.join(".")} is required but got nil" unless field.optional
+        raise EncodeError.new("Field #{field.node.path.join(".")} is required but got nil", column: field.node.path.join(".")) unless field.optional
         field.leaves.each do |col|
           buf = @buffers[col.index]
           buf.defs << parent_def
@@ -488,14 +566,14 @@ module Herringbone
         buf.defs << d
         buf.reps&.<< rep
         begin
-          buf.values << field.column.encoder.call(value)
+          buf.values << @encoders[field.column.index].call(value)
         rescue ArgumentError, TypeError, NoMethodError, RangeError => e
-          raise EncodeError, "Cannot write #{value.inspect} to #{field.column.dotted_path}: #{e.message}"
+          raise EncodeError.new("Cannot write #{value.inspect} to #{field.column.dotted_path}: #{e.message}", column: field.column.dotted_path, value: value)
         end
       when :struct
         field.children.each { |ch| shred(ch, lookup(value, ch.name), d, rep) }
       when :list
-        raise EncodeError, "Expected an Array for #{field.node.path.join(".")}, got #{value.class}" unless value.respond_to?(:each_with_index)
+        raise EncodeError.new("Expected an Array for #{field.node.path.join(".")}, got #{value.class}", column: field.node.path.join("."), value: value) unless value.respond_to?(:each_with_index)
         if value.empty?
           field.leaves.each do |col|
             buf = @buffers[col.index]
@@ -508,7 +586,7 @@ module Herringbone
           end
         end
       when :map
-        raise EncodeError, "Expected a Hash for #{field.node.path.join(".")}, got #{value.class}" unless value.respond_to?(:each_pair) || value.is_a?(Array)
+        raise EncodeError.new("Expected a Hash for #{field.node.path.join(".")}, got #{value.class}", column: field.node.path.join("."), value: value) unless value.respond_to?(:each_pair) || value.is_a?(Array)
         if value.empty?
           field.leaves.each do |col|
             buf = @buffers[col.index]
@@ -532,8 +610,11 @@ module Herringbone
     #
     # @param col [Schema::Column] column being written
     # @param buffer [ColumnBuffer] the column's buffered levels and values
+    # @param codec [Integer] codec id to compress the pages with
+    # @param bloom [Hash{Symbol => Numeric, nil}, nil] bloom filter settings (see
+    #   #bloom_filter_settings), nil for no bloom filter
     # @return [Format::ColumnChunk] chunk with its ColumnMetaData, for the row group
-    def write_column_chunk(col, buffer)
+    def write_column_chunk(col, buffer, codec: @codec, bloom: @bloom_filters[col.dotted_path])
       type = col.type
       path = col.dotted_path
       dict_values = nil
@@ -566,7 +647,7 @@ module Herringbone
           type: Format::PageType::DICTIONARY_PAGE,
           dictionary_page_header: Format::DictionaryPageHeader.new(num_values: dict_values.size, encoding: E::PLAIN)
         )
-        uncompressed_total += write_page(header, plain)
+        uncompressed_total += write_page(header, plain, codec: codec)
       end
 
       data_offset = @pos
@@ -600,10 +681,10 @@ module Herringbone
         rep_bytes = max_rep.positive? ? Encodings::RLE.encode_hybrid(reps, max_rep.bit_length) : "".b
         def_bytes = max_def.positive? ? Encodings::RLE.encode_hybrid(defs, max_def.bit_length) : "".b
         uncompressed_total += if @data_page_version == 1
-          write_data_page_v1(n, rep_bytes, def_bytes, encoded, value_encoding)
+          write_data_page_v1(n, rep_bytes, def_bytes, encoded, value_encoding, codec)
         else
           num_rows = max_rep.zero? ? n : reps.count(0)
-          write_data_page_v2(n, n - non_null, num_rows, rep_bytes, def_bytes, encoded, value_encoding)
+          write_data_page_v2(n, n - non_null, num_rows, rep_bytes, def_bytes, encoded, value_encoding, codec)
         end
         pages << PageInfo.new(page_offset, @pos - page_offset, first_row, n - non_null, non_null, range)
         first_row += max_rep.zero? ? n : reps.count(0)
@@ -616,7 +697,7 @@ module Herringbone
         type: type,
         encodings: encodings.uniq,
         path_in_schema: col.path,
-        codec: @codec,
+        codec: codec,
         num_values: buf.defs.size,
         total_uncompressed_size: uncompressed_total,
         total_compressed_size: @pos - chunk_start,
@@ -625,7 +706,7 @@ module Herringbone
         statistics: statistics_for(col, buf.defs, values || indices.uniq.map { |i| dict_values[i] }, order)
       )
       chunk = Format::ColumnChunk.new(file_offset: chunk_start, meta_data: meta)
-      if (bloom = @bloom_filters[path])
+      if bloom
         @pending_bloom_filters << [meta, build_bloom_filter(col, bloom, dict_values, values)]
       end
       @page_indexes << [chunk, column_index_for(col, pages, order), offset_index_for(pages)]
@@ -656,6 +737,37 @@ module Herringbone
       end)
     end
 
+    # Writes a chunk copied from another file. The pages, the ColumnIndex and the bloom filter
+    # hold no file offsets and go out as they are; the ColumnMetaData and the OffsetIndex do, so
+    # those are rebased onto where the chunk lands in this file.
+    #
+    # @param copy [CopiedChunk] the chunk to copy
+    # @return [Format::ColumnChunk] chunk with its ColumnMetaData, for the row group
+    def copy_column_chunk(copy)
+      start = @pos
+      shift = start - copy.start
+      meta = Format::ColumnMetaData.decode(copy.chunk.meta_data.encode).first
+      meta.data_page_offset += shift
+      dict = meta.dictionary_page_offset
+      # Some writers store 0 when there is no dictionary page
+      meta.dictionary_page_offset = dict&.positive? ? dict + shift : nil
+      meta.index_page_offset += shift if meta.index_page_offset
+      meta.total_compressed_size = copy.bytes.bytesize
+      meta.bloom_filter_offset = meta.bloom_filter_length = nil
+      write_raw(copy.bytes)
+      chunk = Format::ColumnChunk.new(file_offset: start, meta_data: meta)
+      @pending_bloom_filters << [meta, copy.bloom_filter] if copy.bloom_filter
+      offset_index = copy.offset_index && Format::OffsetIndex.new(
+        page_locations: copy.offset_index.page_locations.map do |loc|
+          Format::PageLocation.new(offset: loc.offset + shift, compressed_page_size: loc.compressed_page_size,
+            first_row_index: loc.first_row_index)
+        end,
+        unencoded_byte_array_data_bytes: copy.offset_index.unencoded_byte_array_data_bytes
+      )
+      @page_indexes << [chunk, copy.column_index, offset_index]
+      chunk
+    end
+
     # nil when the column has no defined sort order, or a page's values have no min/max (all NaN)
     #
     # @param col [Schema::Column] column the pages belong to
@@ -691,18 +803,20 @@ module Herringbone
       end
     end
 
-    # Page indexes go after the last row group: all column indexes, then all offset indexes
+    # Page indexes go after the last row group: all column indexes, then all offset indexes.
+    # A copied chunk brings its ColumnIndex already encoded, and may come without either index.
     #
     # @return [void]
     def write_page_indexes
       @page_indexes.each do |chunk, column_index, _|
         next unless column_index
-        bytes = column_index.encode
+        bytes = column_index.is_a?(String) ? column_index : column_index.encode
         chunk.column_index_offset = @pos
         chunk.column_index_length = bytes.bytesize
         write_raw(bytes)
       end
       @page_indexes.each do |chunk, _, offset_index|
+        next unless offset_index
         bytes = offset_index.encode
         chunk.offset_index_offset = @pos
         chunk.offset_index_length = bytes.bytesize
@@ -778,12 +892,13 @@ module Herringbone
       filter
     end
 
-    # Bloom filters go right after the row group's column chunks, in column order
+    # Bloom filters go right after the row group's column chunks, in column order. A copied
+    # chunk brings its filter already encoded.
     #
     # @return [void]
     def write_bloom_filters
       @pending_bloom_filters.each do |meta, filter|
-        bytes = filter.encode
+        bytes = filter.is_a?(String) ? filter : filter.encode
         meta.bloom_filter_offset = @pos
         meta.bloom_filter_length = bytes.bytesize
         write_raw(bytes)
@@ -904,9 +1019,10 @@ module Herringbone
     # @param body [String] uncompressed page body
     # @param compressed [String, nil] bytes to write as the page body, when already prepared (v2 data
     #   pages, whose levels stay uncompressed); +body+ is compressed otherwise
+    # @param codec [Integer] codec id to compress +body+ with
     # @return [Integer]
-    def write_page(header, body, compressed = nil)
-      compressed ||= Compression.compress(@codec, body)
+    def write_page(header, body, compressed = nil, codec: @codec)
+      compressed ||= Compression.compress(codec, body)
       header.uncompressed_page_size ||= body.bytesize
       header.compressed_page_size = compressed.bytesize
       header.crc = Zlib.crc32(compressed).then { |c| (c >= 0x8000_0000) ? c - 0x1_0000_0000 : c }
@@ -923,8 +1039,9 @@ module Herringbone
     # @param def_bytes [String] RLE-encoded definition levels, empty when the column has none
     # @param encoded [String] encoded values
     # @param encoding [Integer] encoding id of the values
+    # @param codec [Integer] codec id to compress the page with
     # @return [Integer] uncompressed size including the header
-    def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding)
+    def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding, codec)
       body = String.new(encoding: Encoding::BINARY)
       body << [rep_bytes.bytesize].pack("V") << rep_bytes unless rep_bytes.empty?
       body << [def_bytes.bytesize].pack("V") << def_bytes unless def_bytes.empty?
@@ -936,7 +1053,7 @@ module Herringbone
           definition_level_encoding: E::RLE, repetition_level_encoding: E::RLE
         )
       )
-      write_page(header, body)
+      write_page(header, body, codec: codec)
     end
 
     # A DATA_PAGE_V2: levels without length prefixes and uncompressed, then the compressed values
@@ -948,9 +1065,10 @@ module Herringbone
     # @param def_bytes [String] RLE-encoded definition levels, empty when the column has none
     # @param encoded [String] encoded values
     # @param encoding [Integer] encoding id of the values
+    # @param codec [Integer] codec id to compress the values with
     # @return [Integer] uncompressed size including the header
-    def write_data_page_v2(n, nulls, rows, rep_bytes, def_bytes, encoded, encoding)
-      compressed = Compression.compress(@codec, encoded)
+    def write_data_page_v2(n, nulls, rows, rep_bytes, def_bytes, encoded, encoding, codec)
+      compressed = Compression.compress(codec, encoded)
       header = Format::PageHeader.new(
         type: Format::PageType::DATA_PAGE_V2,
         uncompressed_page_size: rep_bytes.bytesize + def_bytes.bytesize + encoded.bytesize,
@@ -958,17 +1076,17 @@ module Herringbone
           num_values: n, num_nulls: nulls, num_rows: rows, encoding: encoding,
           definition_levels_byte_length: def_bytes.bytesize,
           repetition_levels_byte_length: rep_bytes.bytesize,
-          is_compressed: @codec != Format::Codec::UNCOMPRESSED
+          is_compressed: codec != Format::Codec::UNCOMPRESSED
         )
       )
-      write_page(header, "".b, rep_bytes + def_bytes + compressed)
+      write_page(header, "".b, rep_bytes + def_bytes + compressed, codec: codec)
     end
 
     # Statistics and column index bounds longer than this are truncated, see #truncate_min
     STAT_TRUNCATE_BYTES = 64
     # Sort key for types whose Ruby ordering already matches Parquet's; compared by identity to
     # skip calling it
-    IDENTITY = ->(v) { v }
+    IDENTITY = Ractor.make_shareable(->(v) { v })
 
     # Chunk statistics: null count, plus min/max (flagged exact unless truncated) when the column
     # has a sort order and non-NaN values