herringbone 0.3.0 → 0.4.0

data/lib/herringbone/thrift.rb

@@ -401,12 +401,12 @@ module Herringbone
       class << self
         # @return [Array<Field>] declared fields, sorted by id
         def fields
-          @fields ||= []
+          @fields || []
         end
 
         # @return [Hash{Integer => Field}] declared fields by id, for decoding
         def fields_by_id
-          @fields_by_id ||= fields.to_h { |f| [f.id, f] }
+          @fields_by_id || {}
         end
 
         # Declares a field and defines its accessor
@@ -417,9 +417,9 @@ module Herringbone
         #   or a Struct subclass
         # @return [void]
         def field(id, name, type)
-          fields << Field.new(id, name, type, :"@#{name}")
-          fields.sort_by!(&:id)
-          @fields_by_id = nil
+          # Shareable, so that structs can be decoded in any Ractor
+          @fields = Ractor.make_shareable((fields + [Field.new(id, name, type, :"@#{name}")]).sort_by(&:id))
+          @fields_by_id = Ractor.make_shareable(@fields.to_h { |f| [f.id, f] })
           attr_accessor name
         end
 

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/herringbone/writer.rb

@@ -56,14 +56,14 @@ module Herringbone
     }.freeze
 
     # Encoding id => physical types it may be used for, per the Parquet encodings spec
-    VALID_ENCODINGS = {
+    VALID_ENCODINGS = Ractor.make_shareable({
       E::PLAIN => T::NAMES.keys,
       E::RLE => [T::BOOLEAN],
       E::DELTA_BINARY_PACKED => [T::INT32, T::INT64],
       E::DELTA_LENGTH_BYTE_ARRAY => [T::BYTE_ARRAY],
       E::DELTA_BYTE_ARRAY => [T::BYTE_ARRAY, T::FIXED_LEN_BYTE_ARRAY],
       E::BYTE_STREAM_SPLIT => [T::INT32, T::INT64, T::FLOAT, T::DOUBLE, T::FIXED_LEN_BYTE_ARRAY]
-    }.freeze
+    })
 
     # Largest dictionary build_dictionary keeps; above it the chunk is written without a dictionary.
     # Byte-array columns are dictionary-encoded by ByteValues, which applies its own
@@ -132,7 +132,7 @@ module Herringbone
     # @param bloom_filters [Boolean, Array<String>, Hash{String => Boolean, Hash}, nil] see the class
     #   description
     # @raise [ArgumentError] for an invalid IO, schema or option
-    # @raise [MissingCodecError] when the codec's optional gem cannot be loaded
+    # @raise [MissingCodecError] when the codec's optional gem is not loaded
     # @raise [UnsupportedError] when the codec is not supported
     def initialize(io, schema, compression: :snappy, row_group_bytes: 16 * 1024 * 1024, row_group_rows: nil,
       page_bytes: 1024 * 1024, page_rows: 20_000, data_page_version: 1, dictionary: true, encodings: {},
@@ -189,20 +189,20 @@ module Herringbone
           if buf.nil?
             shred(field, value, 0, 0)
           elsif value.nil?
-            raise EncodeError, "Field #{name} is required but got nil" unless field.optional
+            raise EncodeError.new("Field #{name} is required but got nil", column: name) unless field.optional
             buf.defs << 0
           else
             begin
               buf.values << encoder.call(value)
             rescue ArgumentError, TypeError, NoMethodError, RangeError => e
-              raise EncodeError, "Cannot write #{value.inspect} to #{name}: #{e.message}"
+              raise EncodeError.new("Cannot write #{value.inspect} to #{name}: #{e.message}", column: name, value: value)
             end
             buf.defs << field.def_level
           end
         end
       rescue EncodeError => e
         rollback_row(marks)
-        raise EncodeError, "Row #{@total_rows + @buffered_rows}: #{e.message}"
+        raise EncodeError.new("Row #{@total_rows + @buffered_rows}: #{e.message}", row: @total_rows + @buffered_rows, column: e.column, value: e.value)
       rescue
         rollback_row(marks)
         raise
@@ -314,10 +314,12 @@ module Herringbone
           col.max_repetition_level.positive? ? String.new(encoding: Encoding::BINARY) : nil,
           new_values_store(col))
       end
+      # Column#encoder builds its Proc on every call
+      @encoders = @schema.columns.map(&:encoder)
       # Top-level, non-repeated leaves are written directly, everything else is shredded
       @plan = @schema.fields.map do |field|
         flat = field.leaf? && field.column.max_repetition_level.zero?
-        [field, field.name, field.name.to_sym, flat ? @buffers[field.column.index] : nil, flat ? field.column.encoder : nil]
+        [field, field.name, field.name.to_sym, flat ? @buffers[field.column.index] : nil, flat ? @encoders[field.column.index] : nil]
       end
       @nested_buffers = @plan.reject { |p| p[3] }.flat_map { |p| p[0].leaves.map { |c| @buffers[c.index] } }
       @buffered_rows = 0
@@ -472,7 +474,7 @@ module Herringbone
     #   map key is nil, or a leaf value cannot be encoded
     def shred(field, value, parent_def, rep)
       if value.nil?
-        raise EncodeError, "Field #{field.node.path.join(".")} is required but got nil" unless field.optional
+        raise EncodeError.new("Field #{field.node.path.join(".")} is required but got nil", column: field.node.path.join(".")) unless field.optional
         field.leaves.each do |col|
           buf = @buffers[col.index]
           buf.defs << parent_def
@@ -488,14 +490,14 @@ module Herringbone
         buf.defs << d
         buf.reps&.<< rep
         begin
-          buf.values << field.column.encoder.call(value)
+          buf.values << @encoders[field.column.index].call(value)
         rescue ArgumentError, TypeError, NoMethodError, RangeError => e
-          raise EncodeError, "Cannot write #{value.inspect} to #{field.column.dotted_path}: #{e.message}"
+          raise EncodeError.new("Cannot write #{value.inspect} to #{field.column.dotted_path}: #{e.message}", column: field.column.dotted_path, value: value)
         end
       when :struct
         field.children.each { |ch| shred(ch, lookup(value, ch.name), d, rep) }
       when :list
-        raise EncodeError, "Expected an Array for #{field.node.path.join(".")}, got #{value.class}" unless value.respond_to?(:each_with_index)
+        raise EncodeError.new("Expected an Array for #{field.node.path.join(".")}, got #{value.class}", column: field.node.path.join("."), value: value) unless value.respond_to?(:each_with_index)
         if value.empty?
           field.leaves.each do |col|
             buf = @buffers[col.index]
@@ -508,7 +510,7 @@ module Herringbone
           end
         end
       when :map
-        raise EncodeError, "Expected a Hash for #{field.node.path.join(".")}, got #{value.class}" unless value.respond_to?(:each_pair) || value.is_a?(Array)
+        raise EncodeError.new("Expected a Hash for #{field.node.path.join(".")}, got #{value.class}", column: field.node.path.join("."), value: value) unless value.respond_to?(:each_pair) || value.is_a?(Array)
         if value.empty?
           field.leaves.each do |col|
             buf = @buffers[col.index]
@@ -968,7 +970,7 @@ module Herringbone
     STAT_TRUNCATE_BYTES = 64
     # Sort key for types whose Ruby ordering already matches Parquet's; compared by identity to
     # skip calling it
-    IDENTITY = ->(v) { v }
+    IDENTITY = Ractor.make_shareable(->(v) { v })
 
     # Chunk statistics: null count, plus min/max (flagged exact unless truncated) when the column
     # has a sort order and non-NaN values

data/lib/herringbone/xxhash.rb

@@ -3,9 +3,9 @@
 module Herringbone
   # XXH64 (seed 0), the hash Parquet bloom filters use.
   #
-  # When the optional "xxhash" gem (a C extension) can be loaded it is used, which is 20-40x
-  # faster; otherwise hashing is pure Ruby. The gem is only a speedup, so nothing fails without
-  # it. +XXHash.backend = :ruby+ forces pure Ruby (for tests and benchmarks).
+  # When the optional "xxhash" gem (a C extension) is loaded (+require "xxhash"+) it is used,
+  # which is 20-40x faster; otherwise hashing is pure Ruby. The gem is only a speedup, so nothing
+  # fails without it. +XXHash.backend = :ruby+ forces pure Ruby (for tests and benchmarks).
   #
   # The pure-Ruby version keeps every 64-bit value as two 32-bit halves, and multiplies by the
   # XXH64 primes split into 16-bit pieces, so that no intermediate result leaves the Fixnum range.
@@ -286,8 +286,7 @@ module Herringbone
 
     module_eval(Generator.source, __FILE__, __LINE__)
 
-    @native = nil # nil: not resolved yet, false: pure Ruby, else the native module
-    @native_lib = nil
+    @native = nil # nil: native when loaded, false: pure Ruby, else the native module
 
     class << self
       # XXH64 of a String's bytes, as an unsigned 64-bit Integer
@@ -296,7 +295,7 @@ module Herringbone
       # @return [Integer] the hash, 0...2^64
       def xxh64(bytes)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? native.xxh64(bytes, 0) : ruby_xxh64(bytes)
       end
 
@@ -307,7 +306,7 @@ module Herringbone
       # @return [Integer] the hash, 0...2^64
       def xxh64_u64(lane)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         return native.xxh64([lane].pack("Q<"), 0) if native
         ruby_xxh64_lane((lane >> 32) & M32, lane & M32)
       end
@@ -318,7 +317,7 @@ module Herringbone
       # @return [Integer] the hash, 0...2^64
       def xxh64_u32(word)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         return native.xxh64([word].pack("L<"), 0) if native
         ruby_xxh64_u32(word & M32)
       end
@@ -329,7 +328,7 @@ module Herringbone
       # @return [Array<Integer>] the hashes, in the same order
       def xxh64_u64_all(lanes)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         if native
           packed = lanes.pack("Q<*")
           Array.new(lanes.size) { |i| native.xxh64(packed.byteslice(i << 3, 8), 0) }
@@ -344,7 +343,7 @@ module Herringbone
       # @return [Array<Integer>] the hashes, in the same order
       def xxh64_u32_all(words)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         if native
           packed = words.pack("L<*")
           Array.new(words.size) { |i| native.xxh64(packed.byteslice(i << 2, 4), 0) }
@@ -359,7 +358,7 @@ module Herringbone
       # @return [Array<Integer>] the hashes, in the same order
       def xxh64_all(strings)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? strings.map { |s| native.xxh64(s, 0) } : strings.map { |s| ruby_xxh64(s) }
       end
 
@@ -368,57 +367,43 @@ module Herringbone
       # @return [Symbol] +:native+ or +:ruby+
       def backend
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? :native : :ruby
       end
 
-      # :ruby forces pure Ruby, :native requires the xxhash gem (UnsupportedError if it cannot be
-      # loaded), nil goes back to the default: native when available
+      # :ruby forces pure Ruby, :native the xxhash gem (UnsupportedError if it is not loaded), nil
+      # goes back to the default: native when loaded
       #
       # @param name [Symbol, nil] +:ruby+, +:native+ or nil
       # @return [void]
-      # @raise [UnsupportedError] for +:native+ when the gem cannot be loaded
+      # @raise [UnsupportedError] for +:native+ when the gem is not loaded
       # @raise [ArgumentError] for any other name
       def backend=(name)
         @native = case name
         when :ruby then false
         when :native
-          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem could not be loaded")
+          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem is not loaded (require \"#{NATIVE_GEM}\")")
         when nil then nil
         else raise ArgumentError, "Unknown XXHash backend #{name.inspect} (expected :ruby, :native or nil)"
         end
       end
 
-      # Whether the native xxhash gem can be loaded (whatever the selected backend)
+      # Whether the native xxhash gem is loaded (whatever the selected backend)
       #
-      # @return [Boolean] true when the gem is loadable
+      # @return [Boolean] true when the gem is loaded
       def native_available?
         !!native_library
       end
 
       private
 
-      # Picks the default backend on first use: native when the gem loads, pure Ruby otherwise
-      #
-      # @return [Module, false] the native module, or false for pure Ruby
-      def resolve_backend
-        @native = native_library || false
-      end
-
-      # Requires the xxhash gem once and memoizes the result (a failed require is not retried)
+      # Not memoized, so that it works the same in every Ractor and once the gem is required later
       #
       # @return [Module, false] the gem's module to call +xxh64(data, seed)+ on, or false when it
-      #   cannot be loaded
+      #   is not loaded
       def native_library
-        if @native_lib.nil?
-          @native_lib = begin
-            require NATIVE_GEM
-            defined?(::XXhash::XXhashInternal) ? ::XXhash::XXhashInternal : ::XXhash
-          rescue LoadError
-            false
-          end
-        end
-        @native_lib
+        return ::XXhash::XXhashInternal if defined?(::XXhash::XXhashInternal)
+        defined?(::XXhash.xxh64) ? ::XXhash : false
       end
     end
   end

data/lib/herringbone.rb

@@ -10,8 +10,33 @@ module Herringbone
   class Error < StandardError; end
   # The file is not valid Parquet (bad metadata, corrupt pages...)
   class FormatError < Error; end
+
   # A value cannot be written to its column
-  class EncodeError < Error; end
+  class EncodeError < Error
+    # @return [Integer, nil] index of the row being written (0 for the first), when known
+    attr_reader :row
+    # @return [String, nil] dotted path of the column the value was meant for, when known
+    attr_reader :column
+    # @return [Object] the value that could not be written (nil when it was a missing required value)
+    attr_reader :value
+
+    # @param message [String, nil] the error message
+    # @param row [Integer, nil] index of the row being written
+    # @param column [String, nil] dotted path of the column
+    # @param value [Object] the value that could not be written
+    def initialize(message = nil, row: nil, column: nil, value: nil)
+      super(message)
+      @row = row
+      @column = column
+      @value = value
+    end
+  end
+
+  # A row does not fit a schema that was inferred from earlier rows (Herringbone.write without
+  # +schema:+, SimpleWriter). The message says what was inferred, from how many rows, and how to
+  # declare the column instead.
+  class SchemaMismatch < EncodeError; end
+
   # The file (or the writer configuration) uses a Parquet feature Herringbone does not implement,
   # such as encryption or a codec whose library is unavailable
   class UnsupportedError < Error; end
@@ -20,6 +45,7 @@ end
 require_relative "herringbone/io_buffer_support"
 require_relative "herringbone/codecs/snappy"
 require_relative "herringbone/codecs/lz4"
+require_relative "herringbone/codecs/lzo"
 require_relative "herringbone/thrift"
 require_relative "herringbone/format"
 require_relative "herringbone/encodings/rle"
@@ -32,6 +58,8 @@ require_relative "herringbone/active_record"
 require_relative "herringbone/reader"
 require_relative "herringbone/byte_values"
 require_relative "herringbone/writer"
+require_relative "herringbone/inferring_writer"
+require_relative "herringbone/simple_writer"
 require_relative "herringbone/xxhash"
 require_relative "herringbone/bloom_filter"
 require_relative "herringbone/inspector"
@@ -43,10 +71,13 @@ module Herringbone
   # Writes +records+ to +io+ (any IO responding to #write; Herringbone never opens files by path)
   # and returns the number of rows written. +records+ is an Enumerable of rows, or an ActiveRecord
   # model or relation, which is read with find_each. Without +schema+, the schema comes from the
-  # model's columns (Schema.from_active_record) or is inferred from the first rows (Schema.infer).
-  # Other options go to Writer.
+  # model's columns (Schema.from_active_record) or is inferred from the first rows (Schema.infer);
+  # fields declared in the block replace inferred ones. +records+ is iterated once, so a source
+  # that can only be read once (a cursor, a lazy Enumerator over an IO) works too. Other options go
+  # to Writer.
   #
   #   File.open("orders.parquet", "wb") { |f| Herringbone.write(f, Order.where(created_at: 1.year.ago..)) }
+  #   Herringbone.write(io, events.lazy.map(&:to_h)) { json :payload }
   #
   # @param io [IO, #write] destination; written sequentially, never closed
   # @param records [Enumerable<Hash, Array, Object>, Class, #find_each] rows (Hashes, Arrays in schema
@@ -66,27 +97,40 @@ module Herringbone
   # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
   # @option options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
   #   columns to write split block bloom filters for, see Writer
+  # @yield optional block for Schema.infer (Builder DSL), declaring fields that replace inferred ones;
+  #   ignored when the schema is not inferred
   # @return [Integer] number of rows written
   # @raise [ArgumentError] when the schema has to be inferred and +records+ is empty or holds Array
   #   rows, or an option is invalid
   # @raise [EncodeError] when a row does not fit the schema
-  def write(io, records, schema: nil, **options)
+  # @raise [SchemaMismatch] when a row does not fit the inferred schema; the file is left unfinished
+  def write(io, records, schema: nil, **options, &overrides)
     model = if records.respond_to?(:klass) then records.klass
     elsif records.respond_to?(:columns) && records.respond_to?(:find_each) then records
     end
-    schema ||= model ? Schema.from_active_record(model) : Schema.infer(records)
-    Writer.open(io, schema, **options) do |writer|
+    schema ||= Schema.from_active_record(model) if model
+    writer = if schema
+      Writer.new(io, schema, **options)
+    else
+      fix = "Herringbone.write(io, rows) { %s }"
+      InferringWriter.new(io, fix: fix, **options) { |sample| Schema.infer(sample, &overrides) }
+    end
+    begin
       if records.respond_to?(:find_each)
         records.find_each { |record| writer << record }
       else
         records.each { |record| writer << record }
       end
-      writer.rows_written
+    rescue Exception # rubocop:disable Lint/RescueException -- also abort on Interrupt
+      writer.abort
+      raise
     end
+    writer.close
+    writer.rows_written
   end
 
   # Compression codecs this process can read and write, e.g. [:none, :snappy, :gzip, :lz4, :lz4_hadoop, :zstd].
-  # :zstd and :brotli are listed when the zstd-ruby / brotli gems can be loaded.
+  # :zstd and :brotli are listed when the zstd-ruby / brotli gems are loaded.
   #
   # @return [Array<Symbol>] codec names accepted by the writer's +compression:+ option
   def codecs

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: herringbone
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 bindir: bin
 cert_chain: []
-date: 2026-10-01 00:00:00.000000000 Z
+date: 2026-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -33,6 +33,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - LICENSE
 - README.md
 - bin/herringbone
@@ -41,12 +42,14 @@ files:
 - lib/herringbone/bloom_filter.rb
 - lib/herringbone/byte_values.rb
 - lib/herringbone/codecs/lz4.rb
+- lib/herringbone/codecs/lzo.rb
 - lib/herringbone/codecs/snappy.rb
 - lib/herringbone/compression.rb
 - lib/herringbone/encodings/delta.rb
 - lib/herringbone/encodings/plain.rb
 - lib/herringbone/encodings/rle.rb
 - lib/herringbone/format.rb
+- lib/herringbone/inferring_writer.rb
 - lib/herringbone/inspector.rb
 - lib/herringbone/io_buffer_support.rb
 - lib/herringbone/reader.rb
@@ -56,6 +59,7 @@ files:
 - lib/herringbone/reader/page_stream.rb
 - lib/herringbone/reader/scan.rb
 - lib/herringbone/schema.rb
+- lib/herringbone/simple_writer.rb
 - lib/herringbone/thrift.rb
 - lib/herringbone/types.rb
 - lib/herringbone/version.rb