herringbone 0.3.0 → 0.4.0

data/lib/herringbone/compression.rb

@@ -4,80 +4,67 @@ require "zlib"
 require "stringio"
 
 module Herringbone
-  # Raised when a file uses (or a writer asks for) a codec whose library is not installed
+  # Raised when a file uses (or a writer asks for) a codec whose library is not loaded
   class MissingCodecError < UnsupportedError
     # @return [String] codec name (+"ZSTD"+) and name of the gem providing it (+"zstd-ruby"+)
     attr_reader :codec, :gem_name
 
     # @param codec [String] codec name, as shown in the message
     # @param gem_name [String] gem to add to the Gemfile
-    # @param load_error [LoadError] the error from requiring the gem, whose message is included
-    def initialize(codec, gem_name, load_error)
+    # @param path [String] what to require to load the gem
+    def initialize(codec, gem_name, path)
       @codec = codec
       @gem_name = gem_name
-      super("#{codec} compression needs the \"#{gem_name}\" gem, which could not be loaded " \
-        "(#{load_error.message}). Add `gem \"#{gem_name}\"` to your Gemfile to use #{codec}.")
+      super("#{codec} compression needs the \"#{gem_name}\" gem, which is not loaded. Add " \
+        "`gem \"#{gem_name}\"` to your Gemfile and `require \"#{path}\"` to use #{codec}.")
     end
   end
 
   # Dispatches page (de)compression by Parquet codec id. Snappy and LZ4 are pure Ruby and GZIP
-  # uses zlib, so those always work. ZSTD and Brotli come from optional gems (zstd-ruby, brotli),
-  # which are required on first use; if they are missing, MissingCodecError says what to add.
-  # (Snappy also uses the optional snappy gem when it is installed, see Codecs::Snappy.)
+  # uses zlib, so those always work. LZO is pure Ruby too, but can only be read. ZSTD and Brotli
+  # come from optional gems (zstd-ruby, brotli), which the application requires; if they are
+  # not loaded, MissingCodecError says what to add.
+  # (Snappy also uses the optional snappy gem when it is loaded, see Codecs::Snappy.)
   module Compression
     module_function
 
     # Codecs backed by optional native gems: codec id => [name, gem, require path, constant]
-    LIBRARIES = {
+    LIBRARIES = Ractor.make_shareable({
       Format::Codec::ZSTD => ["ZSTD", "zstd-ruby", "zstd-ruby", :Zstd],
       Format::Codec::BROTLI => ["Brotli", "brotli", "brotli", :Brotli]
-    }.freeze
-
-    @libraries = {}
-    @library_mutex = Mutex.new
+    })
 
-    # The library module for a codec backed by an optional gem, requiring it on first use
+    # The library module for a codec backed by an optional gem
     #
     # @param codec [Integer] a codec id that is a key of LIBRARIES
     # @return [Module] the gem's module (+Zstd+ or +Brotli+)
-    # @raise [MissingCodecError] when the gem cannot be loaded
+    # @raise [MissingCodecError] when the gem is not loaded
     # @raise [KeyError] for a codec id not in LIBRARIES
     def library(codec)
-      @libraries.fetch(codec) do
-        @library_mutex.synchronize do
-          @libraries.fetch(codec) do
-            name, gem_name, path, const = LIBRARIES.fetch(codec)
-            begin
-              require_library(path)
-            rescue LoadError => e
-              raise MissingCodecError.new(name, gem_name, e)
-            end
-            @libraries[codec] = Object.const_get(const)
-          end
-        end
-      end
+      name, gem_name, path, const = LIBRARIES.fetch(codec)
+      loaded_library(const) || raise(MissingCodecError.new(name, gem_name, path))
     end
 
-    # Requires a codec gem; a separate method so tests can stub it to simulate a missing gem
+    # A separate method so tests can stub it to simulate a missing gem
     #
-    # @param path [String] require path of the gem
-    # @return [Boolean] whether the library was newly loaded
-    # @raise [LoadError] when the gem is not installed
-    def require_library(path)
-      require path
+    # @param const [Symbol] top-level constant the gem defines
+    # @return [Module, nil] the gem's module, nil when it is not loaded
+    def loaded_library(const)
+      Object.const_get(const) if Object.const_defined?(const)
     end
 
     # Raises MissingCodecError (or UnsupportedError) unless +codec+ can be used
     #
     # @param codec [Integer, Symbol, String] codec id or name (see NAMES)
     # @return [Module, nil] the gem's module for a gem-backed codec, nil for a built-in one
-    # @raise [MissingCodecError] when the codec's gem cannot be loaded
-    # @raise [UnsupportedError] for a codec herringbone does not implement (LZO)
+    # @raise [MissingCodecError] when the codec's gem is not loaded
+    # @raise [UnsupportedError] for a codec herringbone cannot write (LZO)
     # @raise [ArgumentError] for an unknown codec name
     def ensure_available!(codec)
       codec = codec_id(codec)
       return library(codec) if LIBRARIES.key?(codec)
       return if SUPPORTED.include?(codec)
+      raise UnsupportedError, "LZO compression is only supported for reading" if codec == Format::Codec::LZO
       raise UnsupportedError, "#{Format::Codec::NAMES[codec] || codec} compression is not supported"
     end
 
@@ -115,7 +102,7 @@ module Herringbone
     # @param uncompressed_size [Integer] expected decompressed size in bytes
     # @return [String] decompressed bytes (binary)
     # @raise [UnsupportedError] for a codec herringbone does not implement
-    # @raise [MissingCodecError] when the codec's gem cannot be loaded
+    # @raise [MissingCodecError] when the codec's gem is not loaded
     # @raise [FormatError] when the decompressed size does not match +uncompressed_size+
     def decompress(codec, data, uncompressed_size)
       return "".b if uncompressed_size.zero? && data.empty?
@@ -125,6 +112,7 @@ module Herringbone
       when Format::Codec::GZIP then gunzip(data)
       when Format::Codec::LZ4_RAW then Codecs::LZ4.decompress_block(data, uncompressed_size)
       when Format::Codec::LZ4 then Codecs::LZ4.decompress_hadoop(data, uncompressed_size)
+      when Format::Codec::LZO then Codecs::LZO.decompress_hadoop(data, uncompressed_size)
       when Format::Codec::ZSTD then library(codec).decompress(data)
       when Format::Codec::BROTLI then library(codec).inflate(data)
       else
@@ -143,7 +131,7 @@ module Herringbone
     # @param data [String] bytes to compress
     # @return [String] compressed bytes (binary)
     # @raise [UnsupportedError] for a codec herringbone does not implement
-    # @raise [MissingCodecError] when the codec's gem cannot be loaded
+    # @raise [MissingCodecError] when the codec's gem is not loaded
     def compress(codec, data)
       case codec
       when Format::Codec::UNCOMPRESSED then data

data/lib/herringbone/encodings/plain.rb

@@ -8,12 +8,12 @@ module Herringbone
       module_function
 
       # Fixed-width numeric types: pack/unpack directive and byte width (all little-endian).
-      FORMATS = {
+      FORMATS = Ractor.make_shareable({
         Format::Type::INT32 => ["l<", 4],
         Format::Type::INT64 => ["q<", 8],
         Format::Type::FLOAT => ["e", 4],
         Format::Type::DOUBLE => ["E", 8]
-      }.freeze
+      })
 
       # Decodes +count+ values of +type+ from +data+ starting at +pos+.
       # Returns [values, new_pos].

data/lib/herringbone/inferring_writer.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Herringbone
+  # Writes rows whose schema is inferred from the rows themselves, reading them only once: the
+  # first Schema::INFER_SAMPLE rows are held back, the schema is built from them, and then they and
+  # every later row go straight to a Writer. Memory is bounded by the sample, however many rows
+  # follow it. Nothing is written to the IO before the schema is known, so a source that can only
+  # be iterated once (a cursor, a queue, a lazy Enumerator over an IO) loses no rows.
+  #
+  # A row that does not fit the inferred schema raises SchemaMismatch and stops the writer: the
+  # file is left unfinished (no footer) rather than written with rows missing.
+  #
+  # Used by Herringbone.write and SimpleWriter.
+  #
+  # @api private
+  class InferringWriter
+    # @param io [IO, #write] destination, passed to Writer.new
+    # @param fix [String] how a caller declares a column, for SchemaMismatch messages; +%s+ is
+    #   replaced by a declaration such as +string :age+
+    # @param options [Hash{Symbol => Object}] Writer options
+    # @option options [Symbol] :compression (:snappy) codec, checked before any row is read
+    #   (the other Writer options are passed on as well)
+    # @yield [sample] builds the schema once the sample is complete (or at #close)
+    # @yieldparam sample [Array] the rows held back, at most Schema::INFER_SAMPLE
+    # @yieldreturn [Schema]
+    # @raise [MissingCodecError] when the codec's optional gem is not loaded
+    def initialize(io, fix:, **options, &schema_for)
+      # Fail before reading any rows if the codec's library is missing
+      Compression.ensure_available!(Compression.codec_id(options.fetch(:compression, :snappy)))
+      @io = io
+      @fix = fix
+      @options = options
+      @schema_for = schema_for
+      @sample = []
+      @sample_size = nil
+      @writer = nil
+      @mismatch = nil
+    end
+
+    # @param row [Object] row to write, in any form the Writer accepts
+    # @return [self]
+    # @raise [SchemaMismatch] when the row does not fit the inferred schema
+    # @raise [Error] after an earlier SchemaMismatch
+    def <<(row)
+      check_usable!
+      if @writer
+        write(row)
+      else
+        @sample << row
+        start if @sample.size >= Schema::INFER_SAMPLE
+      end
+      self
+    end
+
+    # @return [Integer] rows accepted so far, including the ones held back
+    def rows_written = @writer ? @writer.rows_written : @sample.size
+
+    # Writes the held-back rows if the sample never filled up, then finishes the file
+    #
+    # @return [void]
+    # @raise [SchemaMismatch] when a held-back row does not fit the inferred schema
+    # @raise [Error] after an earlier SchemaMismatch
+    def close
+      check_usable!
+      start unless @writer
+      @writer.close
+    end
+
+    # Stops without finishing the file; when the schema was never built nothing was written at all
+    #
+    # @return [void]
+    def abort
+      @writer&.abort
+    end
+
+    private
+
+    # Builds the schema from the held-back rows, opens the Writer and writes them
+    #
+    # @return [void]
+    # @raise [SchemaMismatch] when a held-back row does not fit
+    def start
+      @sample_size = @sample.size
+      @writer = Writer.new(@io, @schema_for.call(@sample), **@options)
+      sample, @sample = @sample, nil
+      sample.each { |row| write(row) }
+    end
+
+    # @param row [Object] row for the Writer
+    # @return [void]
+    # @raise [SchemaMismatch] when the row does not fit; the Writer is aborted
+    def write(row)
+      @writer << row
+    rescue EncodeError => e
+      @writer.abort
+      @mismatch = SchemaMismatch.new(explain(e), row: e.row, column: e.column, value: e.value)
+      raise @mismatch
+    end
+
+    # @return [void]
+    # @raise [Error] after a SchemaMismatch
+    def check_usable!
+      return unless @mismatch
+      raise Error, "This writer stopped at a schema mismatch and cannot continue " \
+        "(the file is unfinished):\n#{@mismatch.message}"
+    end
+
+    # A multi-line explanation of an EncodeError against the inferred schema
+    #
+    # @param error [EncodeError] error raised by the Writer
+    # @return [String]
+    def explain(error)
+      column = error.column && @writer.schema.column(error.column)
+      column ||= error.column && @writer.schema.columns.find { |c| c.dotted_path.start_with?("#{error.column}.") }
+      top = error.column&.split(".")&.first
+      lines = ["Row #{error.row} does not fit the schema inferred from the first #{@sample_size} rows."]
+      lines << "  column:   #{error.column}" if error.column
+      lines << "  inferred: #{describe(column)}" if column
+      lines << "  got:      #{error.value.inspect[0, 200]} (#{error.value.class})" unless error.value.nil?
+      lines << "  error:    #{error.message.sub(/\ARow \d+: /, "")}"
+      lines << ""
+      lines << "Column types are decided from the first #{Schema::INFER_SAMPLE} rows (or all of them, when there are"
+      lines << "fewer), so a later value of a different type cannot be written to the same column."
+      if top
+        lines << "Declare the column with a type that holds every value, for example:"
+        lines << "  #{format(@fix, declaration_for(top, error.value))}"
+      end
+      lines << "or pass a complete schema. The file was left unfinished (no footer)."
+      lines.join("\n")
+    end
+
+    # @param column [Schema::Column] leaf column
+    # @return [String] its type as the Builder DSL names it, e.g. "int64" or "timestamp (micros)"
+    def describe(column)
+      kind, *details = Types.logical_of(column.node)
+      type = case kind
+      when nil then Format::Type::NAMES[column.type].downcase
+      when :integer then "#{details[1] ? "int" : "uint"}#{details[0]}"
+      when :timestamp, :time then "#{kind} (#{details[0]})"
+      when :decimal then "decimal(#{details[1]}, #{details[0]})"
+      else kind.to_s
+      end
+      (column.path.size > 1) ? "#{type} (at #{column.dotted_path})" : type
+    end
+
+    # A Builder DSL declaration whose type can hold +value+, to suggest in the explanation
+    #
+    # @param name [String] top-level field name
+    # @param value [Object] the value that did not fit
+    # @return [String] e.g. "string :age"
+    def declaration_for(name, value)
+      type = case value
+      when Float, BigDecimal, Rational then "double"
+      when Time, DateTime then "timestamp"
+      when Hash, Array then "json"
+      else "string"
+      end
+      "#{type} :#{name}"
+    end
+  end
+end

data/lib/herringbone/reader/column_chunk_reader.rb

@@ -292,8 +292,8 @@ module Herringbone
         rep_len = dh.repetition_levels_byte_length
         def_len = dh.definition_levels_byte_length
         reps = defs = nil
-        reps = PageStream::HybridDecoder.new(body, 0, rep_len, RLE_WIDTH[@max_rep]) if @max_rep.positive?
-        defs = PageStream::HybridDecoder.new(body, rep_len, rep_len + def_len, RLE_WIDTH[@max_def]) if @max_def.positive?
+        reps = PageStream::HybridDecoder.new(body, 0, rep_len, @max_rep.bit_length) if @max_rep.positive?
+        defs = PageStream::HybridDecoder.new(body, rep_len, rep_len + def_len, @max_def.bit_length) if @max_def.positive?
         data = body.byteslice(rep_len + def_len, body.bytesize - rep_len - def_len)
         if dh.is_compressed != false
           data = decompress(data, header.uncompressed_page_size - rep_len - def_len)
@@ -302,9 +302,6 @@ module Herringbone
         PageStream::Page.new(n, defs, reps, values, conv)
       end
 
-      # Bit width of levels up to a max level (memoized Integer#bit_length)
-      RLE_WIDTH = Hash.new { |h, k| h[k] = k.bit_length }
-
       # [decoder, position after the levels]
       #
       # @param data [String] the uncompressed page
@@ -317,7 +314,7 @@ module Herringbone
       # @raise [FormatError] when the RLE length prefix is cut off
       # @raise [UnsupportedError] for any other level encoding
       def level_decoder(data, pos, encoding, max, n)
-        width = RLE_WIDTH[max]
+        width = max.bit_length
         case encoding
         when E::RLE
           len = data.byteslice(pos, 4)&.unpack1("V") or raise FormatError, "Truncated levels"
@@ -332,9 +329,9 @@ module Herringbone
       end
 
       # Physical type => [String#unpack directive, byte width] for PLAIN fixed-width values
-      FIXED_FORMATS = {
+      FIXED_FORMATS = Ractor.make_shareable({
         T::INT32 => ["l<", 4], T::INT64 => ["q<", 8], T::FLOAT => ["e", 4], T::DOUBLE => ["E", 8]
-      }.freeze
+      })
 
       # [value decoder, converter still to apply to its values (nil for dictionary pages)]
       #

data/lib/herringbone/reader/numo.rb

@@ -27,9 +27,6 @@ module Herringbone
       # Shorthand for Format::Type.
       T = Format::Type
 
-      @loaded = false
-      @mutex = Mutex.new
-
       # How a top-level field becomes a Numo array.
       #   kind:      :fixed (a numeric or boolean leaf), :list (list of numbers) or :object
       #   klass:     the Numo class of the result without nulls
@@ -46,29 +43,19 @@ module Herringbone
 
       module_function
 
-      # Requires Numo once per process (thread-safe); a no-op after the first success.
       # @return [void]
-      # @raise [UnsupportedError] if neither numo-narray-alt nor numo-narray can be loaded
+      # @raise [UnsupportedError] if Numo is not loaded
       def load!
-        return if @loaded
-        @mutex.synchronize do
-          next if @loaded
-          begin
-            require_library
-          rescue LoadError => e
-            raise UnsupportedError, "as: :numo needs the \"numo-narray-alt\" gem (or \"numo-narray\"), " \
-              "which could not be loaded (#{e.message}). Add `gem \"numo-narray-alt\"` to your Gemfile " \
-              "to read columns into Numo arrays."
-          end
-          @loaded = true
-        end
+        return if loaded?
+        raise UnsupportedError, "as: :numo needs the \"numo-narray-alt\" gem (or \"numo-narray\"), which " \
+          "is not loaded. Add `gem \"numo-narray-alt\"` to your Gemfile and `require \"numo/narray\"` " \
+          "to read columns into Numo arrays."
       end
 
-      # Separate from load! so tests can stub it to simulate a missing gem.
-      # @return [Boolean] true if the library was loaded by this call
-      # @raise [LoadError] if "numo/narray" is not installed
-      def require_library
-        require "numo/narray"
+      # A separate method so tests can stub it to simulate a missing gem.
+      # @return [Boolean] whether "numo/narray" is loaded
+      def loaded?
+        defined?(::Numo::NArray) ? true : false
       end
 
       # Picks how +field+ is converted: a numeric or boolean leaf, a list of numbers with a single
@@ -200,9 +187,14 @@ module Herringbone
         out
       end
 
-      # Width => Numo vector of 1 << j for each bit j, built lazily; dotting a bit matrix with it
-      # turns rows of bits into integers. Int64 above 30 bits so the sums cannot overflow.
-      POWERS = Hash.new { |h, w| h[w] = ((w > 30) ? Numo::Int64 : Numo::Int32).cast(Array.new(w) { |j| 1 << j }) }
+      # Numo vector of 1 << j for each bit j of +width+, memoized per Ractor; dotting a bit matrix
+      # with it turns rows of bits into integers. Int64 above 30 bits so the sums cannot overflow.
+      # @param width [Integer] bits per value
+      # @return [Numo::Int32, Numo::Int64] the powers of two
+      def powers(width)
+        cache = Ractor.current[:herringbone_numo_powers] ||= {}
+        cache[width] ||= ((width > 30) ? Numo::Int64 : Numo::Int32).cast(Array.new(width) { |j| 1 << j })
+      end
 
       # +count+ bit-packed values of +width+ bits (LSB first) from +data+ at +pos+ as a Numo
       # array, without a Ruby object per value
@@ -224,7 +216,7 @@ module Herringbone
         end
         # Row i of the [count, width] bit matrix holds value i's bits, least significant first
         bits = Numo::Bit.from_binary(bytes, [count * width]).reshape(count, width)
-        pow = POWERS[width]
+        pow = powers(width)
         pow.class.cast(bits).dot(pow)
       end
     end

data/lib/herringbone/reader.rb

@@ -107,7 +107,7 @@ module Herringbone
     # @return [Reader, Enumerator] self, or an Enumerator of batches when no block is given
     # @raise [ArgumentError] for a non-positive +size+, unknown +as+, negative +from+ / +limit+,
     #   or an unknown column in +columns+ or +where+
-    # @raise [UnsupportedError] with +as: :numo+ when no Numo gem can be loaded
+    # @raise [UnsupportedError] with +as: :numo+ when Numo is not loaded
     def each_batch(size = DEFAULT_BATCH_SIZE, columns: nil, as: :rows, where: nil, from: nil, limit: nil)
       return enum_for(:each_batch, size, columns: columns, as: as, where: where, from: from, limit: limit) unless block_given?
       size = Integer(size)
@@ -249,7 +249,7 @@ module Herringbone
     # @return [Array<Hash>, Hash{String, Symbol => Array}, Hash{String, Symbol => Numo::NArray}]
     #   row Hashes (+:rows+), or field name => all values (+:columns+, +:numo+)
     # @raise [ArgumentError] for an unknown +as+, negative +from+ / +limit+ or an unknown column
-    # @raise [UnsupportedError] with +as: :numo+ when no Numo gem can be loaded
+    # @raise [UnsupportedError] with +as: :numo+ when Numo is not loaded
     def read(columns: nil, as: :rows, where: nil, from: nil, limit: nil)
       if as == :numo
         raise ArgumentError, "limit: must not be negative" if limit&.negative?
@@ -315,7 +315,7 @@ module Herringbone
     # @yieldparam batch [Hash{String, Symbol => Numo::NArray}] field name => values of the batch
     # @yieldreturn [void]
     # @return [void]
-    # @raise [UnsupportedError] when no Numo gem can be loaded
+    # @raise [UnsupportedError] when Numo is not loaded
     def each_numo_batch(size, columns, where, from, limit)
       NumoColumns.load!
       symbolize = @symbolize

data/lib/herringbone/schema.rb

@@ -147,21 +147,18 @@ module Herringbone
       # @return [String] path joined with dots, as used for column names in options
       def dotted_path = @path.join(".")
 
-      # Memoized Types.reader_for of the node.
+      # Types.reader_for of the node. Not memoized: a Schema holding Procs could not be made
+      # Ractor-shareable.
       #
       # @return [Proc, Method, nil] converts a physical value into its Ruby value, or nil when the physical
       #   value is used as-is
-      def converter
-        @converter ||= Types.reader_for(@node)
-      end
+      def converter = Types.reader_for(@node)
 
-      # Memoized Types.writer_for of the node.
+      # Types.writer_for of the node. Not memoized, like #converter.
       #
       # @return [Proc, Method] converts a Ruby value into the physical value to store, raising
       #   ArgumentError / TypeError / RangeError for values that do not fit
-      def encoder
-        @encoder ||= Types.writer_for(@node)
-      end
+      def encoder = Types.writer_for(@node)
     end
 
     # A node of the logical tree.
@@ -217,12 +214,10 @@ module Herringbone
       # @return [Boolean] true for a primitive (+:leaf+) field
       def leaf? = @kind == :leaf
 
-      # Memoized lookup of a struct's children; only valid for +:struct+ fields.
+      # Lookup of a struct's children; only valid for +:struct+ fields.
       #
       # @return [Hash{String => Field}] child fields by name
-      def children_by_name
-        @children_by_name ||= @children.to_h { |c| [c.name, c] }
-      end
+      def children_by_name = @children.to_h { |c| [c.name, c] }
     end
 
     # The root Node, the leaf Columns in file order, and the top-level Fields of the logical tree

data/lib/herringbone/simple_writer.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Herringbone
+  # Writes Parquet the way the CSV gem writes CSV: name the columns, then append rows as Arrays.
+  # The column types are inferred from the first Schema::INFER_SAMPLE rows (see Schema.infer), so
+  # no schema has to be declared.
+  #
+  #   File.open("people.parquet", "wb") do |file|
+  #     Herringbone::SimpleWriter.open(file) do |sw|
+  #       sw.headers!(:id, :name, :age)
+  #       sw << [123, "John", 12]
+  #       sw << { id: 124, name: "Jane" } # Hashes work too; missing columns are nulls
+  #     end
+  #   end
+  #
+  # Every column is nullable. A column that is nil in every sampled row becomes a string, and a file
+  # with headers but no rows has only string columns. Only the sample is held in memory; later rows
+  # are written as they come. A row that does not fit the inferred types raises SchemaMismatch,
+  # explaining what was inferred and how to declare the column, and the writer stops there (the
+  # file is left without a footer). Unlike CSV, the headers are mandatory: a Parquet file always
+  # has named columns.
+  #
+  # Columns declared in a block (Schema::Builder DSL) replace inferred ones, e.g. for a column that
+  # holds both numbers and text:
+  #
+  #   Herringbone::SimpleWriter.new(io) { string :code }
+  class SimpleWriter
+    # Opens a writer, yields it and closes it, finishing the file. If the block raises, the file is
+    # left unfinished (no footer), as with Writer.open. To declare columns, use #initialize and #close.
+    #
+    # @param io [IO, #write] destination; written sequentially, never closed
+    # @param options [Hash{Symbol => Object}] Writer options (compression:, row_group_bytes:...)
+    # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
+    # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
+    #   (other Writer options are passed on as well)
+    # @yield [writer] the open writer
+    # @yieldparam writer [SimpleWriter]
+    # @yieldreturn [Object] returned by open
+    # @return [SimpleWriter, Object] the writer without a block, the block's value with one
+    def self.open(io, **options)
+      writer = new(io, **options)
+      return writer unless block_given?
+      begin
+        result = yield writer
+      rescue Exception # rubocop:disable Lint/RescueException -- also abort on Interrupt
+        writer.abort
+        raise
+      end
+      writer.close
+      result
+    end
+
+    # @return [Array<String>, nil] the column names, nil until #headers! is called
+    attr_reader :headers
+
+    # @param io [IO, #write] destination; nothing is written to it until the column types are known
+    # @param options [Hash{Symbol => Object}] Writer options (compression:, row_group_bytes:...)
+    # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
+    # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
+    #   (other Writer options are passed on as well)
+    # @yield optional block evaluated with +instance_eval+ on a Schema::Builder, declaring columns
+    #   (named in #headers!) that replace inferred ones
+    # @raise [MissingCodecError] when the codec's optional gem is not loaded
+    def initialize(io, **options, &overrides)
+      @headers = nil
+      @overrides = overrides
+      fix = "Herringbone::SimpleWriter.new(io) { %s }"
+      @writer = InferringWriter.new(io, fix: fix, **options) { |sample| schema_for(sample) }
+    end
+
+    # Names the columns, in the order row Arrays list their values. Must be called once, before
+    # the first row.
+    #
+    # @param names [Array<String, Symbol>] column names
+    # @return [self]
+    # @raise [ArgumentError] when called twice, with no names, or with a name given twice
+    def headers!(*names)
+      raise ArgumentError, "headers! was already called" if @headers
+      names = names.flatten.map(&:to_s)
+      raise ArgumentError, "headers! needs at least one column name" if names.empty?
+      duplicates = names.tally.select { |_, n| n > 1 }.keys
+      raise ArgumentError, "Duplicate column names: #{duplicates.join(", ")}" unless duplicates.empty?
+      @headers = names.freeze
+      self
+    end
+
+    # Appends a row: an Array with one value per header, in header order, or a Hash keyed by
+    # header (String or Symbol keys; missing columns are nulls).
+    #
+    # @param row [Array, Hash]
+    # @return [self]
+    # @raise [ArgumentError] when #headers! was not called, an Array has the wrong length or a Hash
+    #   has keys that are not headers
+    # @raise [SchemaMismatch] when a value does not fit its column's inferred type
+    def <<(row)
+      raise ArgumentError, "Call headers! with the column names before writing rows" unless @headers
+      @writer << values_of(row)
+      self
+    end
+
+    # @return [Integer] rows written so far, including those held back to infer the column types
+    def rows_written = @writer.rows_written
+
+    # Writes any rows still held back and finishes the file. The IO is not closed.
+    #
+    # @return [void]
+    # @raise [ArgumentError] when #headers! was never called
+    def close
+      raise ArgumentError, "Call headers! with the column names before closing" unless @headers
+      @writer.close
+    end
+
+    # Stops without finishing the file. What was already written stays in the IO.
+    #
+    # @return [void]
+    def abort = @writer.abort
+
+    private
+
+    # @param row [Array, Hash] row given to #<<
+    # @return [Array] its values in header order
+    # @raise [ArgumentError] when it does not match the headers
+    def values_of(row)
+      case row
+      when Array
+        unless row.size == @headers.size
+          raise ArgumentError, "Expected #{@headers.size} values (#{@headers.join(", ")}), got #{row.size}"
+        end
+        row
+      when Hash
+        unknown = row.keys.map(&:to_s) - @headers
+        raise ArgumentError, "Not among the headers (#{@headers.join(", ")}): #{unknown.join(", ")}" unless unknown.empty?
+        @headers.map { |name| row.fetch(name) { row[name.to_sym] } }
+      else
+        raise ArgumentError, "Rows must be Arrays or Hashes, got #{row.class}"
+      end
+    end
+
+    # @param sample [Array<Array>] held-back rows, values in header order
+    # @return [Schema] inferred from the sample; string columns when it is empty
+    # @raise [ArgumentError] when a column cannot be inferred or a declared column is not a header
+    def schema_for(sample)
+      rows = sample.empty? ? [@headers.to_h { |name| [name, nil] }] : sample.map { |values| @headers.zip(values).to_h }
+      schema = Schema.infer(rows, &@overrides)
+      extra = schema.fields.map(&:name) - @headers
+      raise ArgumentError, "Declared columns not among the headers: #{extra.join(", ")}" unless extra.empty?
+      schema
+    end
+  end
+end