herringbone 0.2.0 → 0.4.0

data/lib/herringbone/xxhash.rb

@@ -3,9 +3,9 @@
 module Herringbone
   # XXH64 (seed 0), the hash Parquet bloom filters use.
   #
-  # When the optional "xxhash" gem (a C extension) can be loaded it is used, which is 20-40x
-  # faster; otherwise hashing is pure Ruby. The gem is only a speedup, so nothing fails without
-  # it. +XXHash.backend = :ruby+ forces pure Ruby (for tests and benchmarks).
+  # When the optional "xxhash" gem (a C extension) is loaded (+require "xxhash"+) it is used,
+  # which is 20-40x faster; otherwise hashing is pure Ruby. The gem is only a speedup, so nothing
+  # fails without it. +XXHash.backend = :ruby+ forces pure Ruby (for tests and benchmarks).
   #
   # The pure-Ruby version keeps every 64-bit value as two 32-bit halves, and multiplies by the
   # XXH64 primes split into 16-bit pieces, so that no intermediate result leaves the Fixnum range.
@@ -15,13 +15,21 @@ module Herringbone
   # arithmetic is written once with the small code generators below, whose output is inlined into
   # the hashing methods: no method calls or allocations in the hot paths.
   module XXHash
+    # 64-bit mask
     M = 0xFFFF_FFFF_FFFF_FFFF
+    # XXH64 PRIME64_1
     P1 = 11_400_714_785_074_694_791
+    # XXH64 PRIME64_2
     P2 = 14_029_467_366_897_019_727
+    # XXH64 PRIME64_3
     P3 = 1_609_587_929_392_839_161
+    # XXH64 PRIME64_4
     P4 = 9_650_029_242_287_828_579
+    # XXH64 PRIME64_5
     P5 = 2_870_177_450_012_600_261
+    # 32-bit mask
     M32 = 0xFFFF_FFFF
+    # unpack format splitting the input into little-endian 32-bit words
     WORDS = "V*" # frozen, unlike a literal in the generated code
 
     # Gem providing the native implementation
@@ -35,8 +43,14 @@ module Herringbone
       # Shifts are written as multiplications and divisions by powers of two, which YARV has
       # specialized instructions for (<< and >> on Integers are method calls).
       #
-      # (hi:lo) = (hi:lo) * c mod 2**64. Products are at most 48 bits wide: the low halves are
+      # (hi:lo) = (hi:lo) * c mod 2^64. Products are at most 48 bits wide: the low halves are
       # multiplied by 16-bit pieces of c, and only the low 32 bits of the cross terms are kept.
+      #
+      # @param hi [String] variable holding the high 32 bits
+      # @param lo [String] variable holding the low 32 bits
+      # @param c [Integer] unsigned 64-bit constant multiplier
+      # @param hi_zero [Boolean] whether +hi+ is known to be 0, which drops its cross terms
+      # @return [String] Ruby source (uses t0 and t1 as scratch)
       def mul(hi, lo, c, hi_zero: false)
         ch = c >> 32
         cl = c & M32
@@ -51,6 +65,11 @@ module Herringbone
       end
 
       # Rotate (hi:lo) left by r bits (0 < r < 32)
+      #
+      # @param hi [String] variable holding the high 32 bits
+      # @param lo [String] variable holding the low 32 bits
+      # @param r [Integer] rotation in bits, 1..31
+      # @return [String] Ruby source (uses t0 as scratch)
       def rotl(hi, lo, r)
         mask = (1 << (32 - r)) - 1
         <<~RUBY
@@ -61,6 +80,12 @@ module Herringbone
       end
 
       # (hi:lo) += (a_hi:a_lo), where the addend is two expressions (constants or variables)
+      #
+      # @param hi [String] variable holding the high 32 bits
+      # @param lo [String] variable holding the low 32 bits
+      # @param a_hi [String, Integer] expression for the addend's high 32 bits
+      # @param a_lo [String, Integer] expression for the addend's low 32 bits
+      # @return [String] Ruby source
       def add(hi, lo, a_hi, a_lo)
         <<~RUBY
           #{lo} += #{a_lo}
@@ -69,43 +94,82 @@ module Herringbone
         RUBY
       end
 
+      # (hi:lo) += c, mod 2^64
+      #
+      # @param hi [String] variable holding the high 32 bits
+      # @param lo [String] variable holding the low 32 bits
+      # @param c [Integer] unsigned 64-bit constant addend
+      # @return [String] Ruby source
       def add_const(hi, lo, c)
         add(hi, lo, c >> 32, c & M32)
       end
 
       # Assigns the 64-bit constant c to (hi:lo)
+      #
+      # @param hi [String] variable for the high 32 bits
+      # @param lo [String] variable for the low 32 bits
+      # @param c [Integer] unsigned 64-bit constant
+      # @return [String] Ruby source
       def set(hi, lo, c)
         "#{hi} = #{c >> 32}\n#{lo} = #{c & M32}\n"
       end
 
       # XXH64 round with a zero accumulator: (hi:lo) = rotl(lane * P2, 31) * P1
+      #
+      # @param hi [String] variable holding the lane's high 32 bits, replaced by the result's
+      # @param lo [String] variable holding the lane's low 32 bits, replaced by the result's
+      # @return [String] Ruby source
       def round0(hi, lo)
         mul(hi, lo, P2) + rotl(hi, lo, 31) + mul(hi, lo, P1)
       end
 
       # Stripe round: acc = rotl(acc + lane * P2, 31) * P1, with the lane in (xh:xl)
+      #
+      # @param hi [String] variable holding the accumulator's high 32 bits
+      # @param lo [String] variable holding the accumulator's low 32 bits
+      # @param xh [String] variable holding the lane's high 32 bits (clobbered)
+      # @param xl [String] variable holding the lane's low 32 bits (clobbered)
+      # @return [String] Ruby source
       def round(hi, lo, xh, xl)
         mul(xh, xl, P2) + add(hi, lo, xh, xl) + rotl(hi, lo, 31) + mul(hi, lo, P1)
       end
 
       # h ^= rotl(v * P2, 31) * P1; h = h * P1 + P4, with v in (vh:vl) (left unchanged)
+      #
+      # @param hi [String] variable holding h's high 32 bits
+      # @param lo [String] variable holding h's low 32 bits
+      # @param vh [String] variable holding the accumulator v's high 32 bits
+      # @param vl [String] variable holding the accumulator v's low 32 bits
+      # @return [String] Ruby source (uses xh and xl as scratch)
       def merge_round(hi, lo, vh, vl)
         "xh = #{vh}\nxl = #{vl}\n" + round0("xh", "xl") +
           "#{hi} ^= xh\n#{lo} ^= xl\n" + mul(hi, lo, P1) + add_const(hi, lo, P4)
       end
 
       # Consumes an 8-byte lane in (xh:xl)
+      #
+      # @param hi [String] variable holding the hash's high 32 bits
+      # @param lo [String] variable holding the hash's low 32 bits
+      # @return [String] Ruby source
       def lane8(hi, lo)
         round0("xh", "xl") + "#{hi} ^= xh\n#{lo} ^= xl\n" + rotl(hi, lo, 27) + mul(hi, lo, P1) + add_const(hi, lo, P4)
       end
 
       # Consumes a 4-byte word in xl
+      #
+      # @param hi [String] variable holding the hash's high 32 bits
+      # @param lo [String] variable holding the hash's low 32 bits
+      # @return [String] Ruby source (sets xh)
       def lane4(hi, lo)
         "xh = 0\n" + mul("xh", "xl", P1, hi_zero: true) + "#{hi} ^= xh\n#{lo} ^= xl\n" +
           rotl(hi, lo, 23) + mul(hi, lo, P2) + add_const(hi, lo, P3)
       end
 
-      # Consumes one byte in xl (as it is below 2**16, byte * P5 needs no splitting)
+      # Consumes one byte in xl (as it is below 2^16, byte * P5 needs no splitting)
+      #
+      # @param hi [String] variable holding the hash's high 32 bits
+      # @param lo [String] variable holding the hash's low 32 bits
+      # @return [String] Ruby source
       def lane1(hi, lo)
         <<~RUBY + rotl(hi, lo, 11) + mul(hi, lo, P1)
           t0 = xl * #{P5 & M32}
@@ -115,12 +179,20 @@ module Herringbone
       end
 
       # Final mix; evaluates to the hash as one Integer
+      #
+      # @param hi [String] variable holding the hash's high 32 bits
+      # @param lo [String] variable holding the hash's low 32 bits
+      # @return [String] Ruby source whose last expression is the 64-bit hash
       def avalanche(hi, lo)
         "#{lo} ^= #{hi} / 2\n" + mul(hi, lo, P2) +
           "#{lo} ^= (#{hi} & 0x1FFFFFFF) * 8 | #{lo} / 536870912\n#{hi} ^= #{hi} / 536870912\n" +
           mul(hi, lo, P3) + "#{lo} ^= #{hi}\n(#{hi} << 32) | #{lo}\n"
       end
 
+      # Source of the pure-Ruby hashing methods, evaluated into XXHash: +ruby_xxh64(bytes)+,
+      # +ruby_xxh64_lane(xh, xl)+ (8 bytes as two 32-bit halves) and +ruby_xxh64_u32(xl)+ (4 bytes)
+      #
+      # @return [String] Ruby source defining the three singleton methods
       def source
         g = self
         <<~RUBY
@@ -214,38 +286,49 @@ module Herringbone
 
     module_eval(Generator.source, __FILE__, __LINE__)
 
-    @native = nil # nil: not resolved yet, false: pure Ruby, else the native module
-    @native_lib = nil
+    @native = nil # nil: native when loaded, false: pure Ruby, else the native module
 
     class << self
       # XXH64 of a String's bytes, as an unsigned 64-bit Integer
+      #
+      # @param bytes [String] data to hash (its encoding is ignored)
+      # @return [Integer] the hash, 0...2^64
       def xxh64(bytes)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? native.xxh64(bytes, 0) : ruby_xxh64(bytes)
       end
 
       # XXH64 of 8 bytes given as a little-endian 64-bit Integer (an INT64 or DOUBLE's PLAIN
       # encoding), signed or unsigned: only its low 64 bits are used
+      #
+      # @param lane [Integer] value whose low 64 bits are hashed
+      # @return [Integer] the hash, 0...2^64
       def xxh64_u64(lane)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         return native.xxh64([lane].pack("Q<"), 0) if native
         ruby_xxh64_lane((lane >> 32) & M32, lane & M32)
       end
 
       # XXH64 of 4 bytes given as a little-endian 32-bit Integer (INT32, FLOAT), signed or unsigned
+      #
+      # @param word [Integer] value whose low 32 bits are hashed
+      # @return [Integer] the hash, 0...2^64
       def xxh64_u32(word)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         return native.xxh64([word].pack("L<"), 0) if native
         ruby_xxh64_u32(word & M32)
       end
 
       # Hashes of many 64-bit Integers (low 64 bits of each)
+      #
+      # @param lanes [Array<Integer>] values to hash, signed or unsigned
+      # @return [Array<Integer>] the hashes, in the same order
       def xxh64_u64_all(lanes)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         if native
           packed = lanes.pack("Q<*")
           Array.new(lanes.size) { |i| native.xxh64(packed.byteslice(i << 3, 8), 0) }
@@ -255,9 +338,12 @@ module Herringbone
       end
 
       # Hashes of many 32-bit Integers (low 32 bits of each)
+      #
+      # @param words [Array<Integer>] values to hash, signed or unsigned
+      # @return [Array<Integer>] the hashes, in the same order
       def xxh64_u32_all(words)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         if native
           packed = words.pack("L<*")
           Array.new(words.size) { |i| native.xxh64(packed.byteslice(i << 2, 4), 0) }
@@ -267,52 +353,57 @@ module Herringbone
       end
 
       # Hashes of many Strings
+      #
+      # @param strings [Array<String>] values whose bytes are hashed
+      # @return [Array<Integer>] the hashes, in the same order
       def xxh64_all(strings)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? strings.map { |s| native.xxh64(s, 0) } : strings.map { |s| ruby_xxh64(s) }
       end
 
       # :native when the xxhash gem is used, :ruby otherwise
+      #
+      # @return [Symbol] +:native+ or +:ruby+
       def backend
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? :native : :ruby
       end
 
-      # :ruby forces pure Ruby, :native requires the xxhash gem (UnsupportedError if it cannot be
-      # loaded), nil goes back to the default: native when available
+      # :ruby forces pure Ruby, :native the xxhash gem (UnsupportedError if it is not loaded), nil
+      # goes back to the default: native when loaded
+      #
+      # @param name [Symbol, nil] +:ruby+, +:native+ or nil
+      # @return [void]
+      # @raise [UnsupportedError] for +:native+ when the gem is not loaded
+      # @raise [ArgumentError] for any other name
       def backend=(name)
         @native = case name
         when :ruby then false
         when :native
-          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem could not be loaded")
+          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem is not loaded (require \"#{NATIVE_GEM}\")")
         when nil then nil
         else raise ArgumentError, "Unknown XXHash backend #{name.inspect} (expected :ruby, :native or nil)"
         end
       end
 
-      # Whether the native xxhash gem can be loaded (whatever the selected backend)
+      # Whether the native xxhash gem is loaded (whatever the selected backend)
+      #
+      # @return [Boolean] true when the gem is loaded
       def native_available?
         !!native_library
       end
 
       private
 
-      def resolve_backend
-        @native = native_library || false
-      end
-
+      # Not memoized, so that it works the same in every Ractor and once the gem is required later
+      #
+      # @return [Module, false] the gem's module to call +xxh64(data, seed)+ on, or false when it
+      #   is not loaded
       def native_library
-        if @native_lib.nil?
-          @native_lib = begin
-            require NATIVE_GEM
-            defined?(::XXhash::XXhashInternal) ? ::XXhash::XXhashInternal : ::XXhash
-          rescue LoadError
-            false
-          end
-        end
-        @native_lib
+        return ::XXhash::XXhashInternal if defined?(::XXhash::XXhashInternal)
+        defined?(::XXhash.xxh64) ? ::XXhash : false
       end
     end
   end

data/lib/herringbone.rb

@@ -6,17 +6,46 @@ require_relative "herringbone/version"
 
 # Pure-Ruby reader and writer for Apache Parquet files
 module Herringbone
+  # Base class of every error Herringbone raises on purpose
   class Error < StandardError; end
   # The file is not valid Parquet (bad metadata, corrupt pages...)
   class FormatError < Error; end
+
   # A value cannot be written to its column
-  class EncodeError < Error; end
+  class EncodeError < Error
+    # @return [Integer, nil] index of the row being written (0 for the first), when known
+    attr_reader :row
+    # @return [String, nil] dotted path of the column the value was meant for, when known
+    attr_reader :column
+    # @return [Object] the value that could not be written (nil when it was a missing required value)
+    attr_reader :value
+
+    # @param message [String, nil] the error message
+    # @param row [Integer, nil] index of the row being written
+    # @param column [String, nil] dotted path of the column
+    # @param value [Object] the value that could not be written
+    def initialize(message = nil, row: nil, column: nil, value: nil)
+      super(message)
+      @row = row
+      @column = column
+      @value = value
+    end
+  end
+
+  # A row does not fit a schema that was inferred from earlier rows (Herringbone.write without
+  # +schema:+, SimpleWriter). The message says what was inferred, from how many rows, and how to
+  # declare the column instead.
+  class SchemaMismatch < EncodeError; end
+
+  # The file (or the writer configuration) uses a Parquet feature Herringbone does not implement,
+  # such as encryption or a codec whose library is unavailable
   class UnsupportedError < Error; end
 end
 
 require_relative "herringbone/io_buffer_support"
 require_relative "herringbone/codecs/snappy"
 require_relative "herringbone/codecs/lz4"
+require_relative "herringbone/codecs/lzo"
 require_relative "herringbone/thrift"
 require_relative "herringbone/format"
 require_relative "herringbone/encodings/rle"
@@ -29,6 +58,8 @@ require_relative "herringbone/active_record"
 require_relative "herringbone/reader"
 require_relative "herringbone/byte_values"
 require_relative "herringbone/writer"
+require_relative "herringbone/inferring_writer"
+require_relative "herringbone/simple_writer"
 require_relative "herringbone/xxhash"
 require_relative "herringbone/bloom_filter"
 require_relative "herringbone/inspector"
@@ -40,27 +71,68 @@ module Herringbone
   # Writes +records+ to +io+ (any IO responding to #write; Herringbone never opens files by path)
   # and returns the number of rows written. +records+ is an Enumerable of rows, or an ActiveRecord
   # model or relation, which is read with find_each. Without +schema+, the schema comes from the
-  # model's columns (Schema.from_active_record) or is inferred from the first rows (Schema.infer).
-  # Other options go to Writer.
+  # model's columns (Schema.from_active_record) or is inferred from the first rows (Schema.infer);
+  # fields declared in the block replace inferred ones. +records+ is iterated once, so a source
+  # that can only be read once (a cursor, a lazy Enumerator over an IO) works too. Other options go
+  # to Writer.
   #
   #   File.open("orders.parquet", "wb") { |f| Herringbone.write(f, Order.where(created_at: 1.year.ago..)) }
-  def write(io, records, schema: nil, **options)
+  #   Herringbone.write(io, events.lazy.map(&:to_h)) { json :payload }
+  #
+  # @param io [IO, #write] destination; written sequentially, never closed
+  # @param records [Enumerable<Hash, Array, Object>, Class, #find_each] rows (Hashes, Arrays in schema
+  #   order, or objects responding to #attributes or #to_h), or an ActiveRecord model or relation
+  # @param schema [Schema, nil] schema to write with; derived from +records+ when nil
+  # @param options [Hash{Symbol => Object}] passed to Writer.new
+  # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+  # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
+  # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
+  # @option options [Integer] :page_bytes (1MB) approximate uncompressed data page size
+  # @option options [Integer] :page_rows (20_000) maximum rows per data page
+  # @option options [Integer] :data_page_version (1) 1 or 2
+  # @option options [Boolean, Array<String>] :dictionary (true) dictionary-encode all eligible columns,
+  #   none, or only the listed dotted column paths
+  # @option options [Hash{String => Symbol}] :encodings ({}) dotted column path => value encoding
+  #   for non-dictionary pages
+  # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
+  # @option options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
+  #   columns to write split block bloom filters for, see Writer
+  # @yield optional block for Schema.infer (Builder DSL), declaring fields that replace inferred ones;
+  #   ignored when the schema is not inferred
+  # @return [Integer] number of rows written
+  # @raise [ArgumentError] when the schema has to be inferred and +records+ is empty or holds Array
+  #   rows, or an option is invalid
+  # @raise [EncodeError] when a row does not fit the schema
+  # @raise [SchemaMismatch] when a row does not fit the inferred schema; the file is left unfinished
+  def write(io, records, schema: nil, **options, &overrides)
     model = if records.respond_to?(:klass) then records.klass
     elsif records.respond_to?(:columns) && records.respond_to?(:find_each) then records
     end
-    schema ||= model ? Schema.from_active_record(model) : Schema.infer(records)
-    Writer.open(io, schema, **options) do |writer|
+    schema ||= Schema.from_active_record(model) if model
+    writer = if schema
+      Writer.new(io, schema, **options)
+    else
+      fix = "Herringbone.write(io, rows) { %s }"
+      InferringWriter.new(io, fix: fix, **options) { |sample| Schema.infer(sample, &overrides) }
+    end
+    begin
       if records.respond_to?(:find_each)
         records.find_each { |record| writer << record }
       else
         records.each { |record| writer << record }
       end
-      writer.rows_written
+    rescue Exception # rubocop:disable Lint/RescueException -- also abort on Interrupt
+      writer.abort
+      raise
     end
+    writer.close
+    writer.rows_written
   end
 
   # Compression codecs this process can read and write, e.g. [:none, :snappy, :gzip, :lz4, :lz4_hadoop, :zstd].
-  # :zstd and :brotli are listed when the zstd-ruby / brotli gems can be loaded.
+  # :zstd and :brotli are listed when the zstd-ruby / brotli gems are loaded.
+  #
+  # @return [Array<Symbol>] codec names accepted by the writer's +compression:+ option
   def codecs
     Compression::NAMES.values.select do |name|
       Compression.ensure_available!(name)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: herringbone
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 bindir: bin
 cert_chain: []
-date: 2026-09-30 00:00:00.000000000 Z
+date: 2026-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -33,6 +33,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - LICENSE
 - README.md
 - bin/herringbone
@@ -41,20 +42,24 @@ files:
 - lib/herringbone/bloom_filter.rb
 - lib/herringbone/byte_values.rb
 - lib/herringbone/codecs/lz4.rb
+- lib/herringbone/codecs/lzo.rb
 - lib/herringbone/codecs/snappy.rb
 - lib/herringbone/compression.rb
 - lib/herringbone/encodings/delta.rb
 - lib/herringbone/encodings/plain.rb
 - lib/herringbone/encodings/rle.rb
 - lib/herringbone/format.rb
+- lib/herringbone/inferring_writer.rb
 - lib/herringbone/inspector.rb
 - lib/herringbone/io_buffer_support.rb
 - lib/herringbone/reader.rb
 - lib/herringbone/reader/column_chunk_reader.rb
 - lib/herringbone/reader/column_cursor.rb
+- lib/herringbone/reader/numo.rb
 - lib/herringbone/reader/page_stream.rb
 - lib/herringbone/reader/scan.rb
 - lib/herringbone/schema.rb
+- lib/herringbone/simple_writer.rb
 - lib/herringbone/thrift.rb
 - lib/herringbone/types.rb
 - lib/herringbone/version.rb