herringbone 0.3.0 → 0.5.0

data/lib/herringbone/xxhash.rb

@@ -3,9 +3,9 @@
 module Herringbone
   # XXH64 (seed 0), the hash Parquet bloom filters use.
   #
-  # When the optional "xxhash" gem (a C extension) can be loaded it is used, which is 20-40x
-  # faster; otherwise hashing is pure Ruby. The gem is only a speedup, so nothing fails without
-  # it. +XXHash.backend = :ruby+ forces pure Ruby (for tests and benchmarks).
+  # When the optional "xxhash" gem (a C extension) is loaded (+require "xxhash"+) it is used,
+  # which is 20-40x faster; otherwise hashing is pure Ruby. The gem is only a speedup, so nothing
+  # fails without it. +XXHash.backend = :ruby+ forces pure Ruby (for tests and benchmarks).
   #
   # The pure-Ruby version keeps every 64-bit value as two 32-bit halves, and multiplies by the
   # XXH64 primes split into 16-bit pieces, so that no intermediate result leaves the Fixnum range.
@@ -286,8 +286,7 @@ module Herringbone
 
     module_eval(Generator.source, __FILE__, __LINE__)
 
-    @native = nil # nil: not resolved yet, false: pure Ruby, else the native module
-    @native_lib = nil
+    @native = nil # nil: native when loaded, false: pure Ruby, else the native module
 
     class << self
       # XXH64 of a String's bytes, as an unsigned 64-bit Integer
@@ -296,7 +295,7 @@ module Herringbone
       # @return [Integer] the hash, 0...2^64
       def xxh64(bytes)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? native.xxh64(bytes, 0) : ruby_xxh64(bytes)
       end
 
@@ -307,7 +306,7 @@ module Herringbone
       # @return [Integer] the hash, 0...2^64
       def xxh64_u64(lane)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         return native.xxh64([lane].pack("Q<"), 0) if native
         ruby_xxh64_lane((lane >> 32) & M32, lane & M32)
       end
@@ -318,7 +317,7 @@ module Herringbone
       # @return [Integer] the hash, 0...2^64
       def xxh64_u32(word)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         return native.xxh64([word].pack("L<"), 0) if native
         ruby_xxh64_u32(word & M32)
       end
@@ -329,7 +328,7 @@ module Herringbone
       # @return [Array<Integer>] the hashes, in the same order
       def xxh64_u64_all(lanes)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         if native
           packed = lanes.pack("Q<*")
           Array.new(lanes.size) { |i| native.xxh64(packed.byteslice(i << 3, 8), 0) }
@@ -344,7 +343,7 @@ module Herringbone
       # @return [Array<Integer>] the hashes, in the same order
       def xxh64_u32_all(words)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         if native
           packed = words.pack("L<*")
           Array.new(words.size) { |i| native.xxh64(packed.byteslice(i << 2, 4), 0) }
@@ -359,7 +358,7 @@ module Herringbone
       # @return [Array<Integer>] the hashes, in the same order
       def xxh64_all(strings)
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? strings.map { |s| native.xxh64(s, 0) } : strings.map { |s| ruby_xxh64(s) }
       end
 
@@ -368,57 +367,43 @@ module Herringbone
       # @return [Symbol] +:native+ or +:ruby+
       def backend
         native = @native
-        native = resolve_backend if native.nil?
+        native = native_library if native.nil?
         native ? :native : :ruby
       end
 
-      # :ruby forces pure Ruby, :native requires the xxhash gem (UnsupportedError if it cannot be
-      # loaded), nil goes back to the default: native when available
+      # :ruby forces pure Ruby, :native the xxhash gem (UnsupportedError if it is not loaded), nil
+      # goes back to the default: native when loaded
       #
       # @param name [Symbol, nil] +:ruby+, +:native+ or nil
       # @return [void]
-      # @raise [UnsupportedError] for +:native+ when the gem cannot be loaded
+      # @raise [UnsupportedError] for +:native+ when the gem is not loaded
       # @raise [ArgumentError] for any other name
       def backend=(name)
         @native = case name
         when :ruby then false
         when :native
-          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem could not be loaded")
+          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem is not loaded (require \"#{NATIVE_GEM}\")")
         when nil then nil
         else raise ArgumentError, "Unknown XXHash backend #{name.inspect} (expected :ruby, :native or nil)"
         end
       end
 
-      # Whether the native xxhash gem can be loaded (whatever the selected backend)
+      # Whether the native xxhash gem is loaded (whatever the selected backend)
       #
-      # @return [Boolean] true when the gem is loadable
+      # @return [Boolean] true when the gem is loaded
       def native_available?
         !!native_library
       end
 
       private
 
-      # Picks the default backend on first use: native when the gem loads, pure Ruby otherwise
-      #
-      # @return [Module, false] the native module, or false for pure Ruby
-      def resolve_backend
-        @native = native_library || false
-      end
-
-      # Requires the xxhash gem once and memoizes the result (a failed require is not retried)
+      # Not memoized, so that it works the same in every Ractor and once the gem is required later
       #
       # @return [Module, false] the gem's module to call +xxh64(data, seed)+ on, or false when it
-      #   cannot be loaded
+      #   is not loaded
       def native_library
-        if @native_lib.nil?
-          @native_lib = begin
-            require NATIVE_GEM
-            defined?(::XXhash::XXhashInternal) ? ::XXhash::XXhashInternal : ::XXhash
-          rescue LoadError
-            false
-          end
-        end
-        @native_lib
+        return ::XXhash::XXhashInternal if defined?(::XXhash::XXhashInternal)
+        defined?(::XXhash.xxh64) ? ::XXhash : false
       end
     end
   end

data/lib/herringbone.rb

@@ -10,8 +10,33 @@ module Herringbone
   class Error < StandardError; end
   # The file is not valid Parquet (bad metadata, corrupt pages...)
   class FormatError < Error; end
+
   # A value cannot be written to its column
-  class EncodeError < Error; end
+  class EncodeError < Error
+    # @return [Integer, nil] index of the row being written (0 for the first), when known
+    attr_reader :row
+    # @return [String, nil] dotted path of the column the value was meant for, when known
+    attr_reader :column
+    # @return [Object] the value that could not be written (nil when it was a missing required value)
+    attr_reader :value
+
+    # @param message [String, nil] the error message
+    # @param row [Integer, nil] index of the row being written
+    # @param column [String, nil] dotted path of the column
+    # @param value [Object] the value that could not be written
+    def initialize(message = nil, row: nil, column: nil, value: nil)
+      super(message)
+      @row = row
+      @column = column
+      @value = value
+    end
+  end
+
+  # A row does not fit a schema that was inferred from earlier rows (Herringbone.write without
+  # +schema:+, SimpleWriter). The message says what was inferred, from how many rows, and how to
+  # declare the column instead.
+  class SchemaMismatch < EncodeError; end
+
   # The file (or the writer configuration) uses a Parquet feature Herringbone does not implement,
   # such as encryption or a codec whose library is unavailable
   class UnsupportedError < Error; end
@@ -20,6 +45,7 @@ end
 require_relative "herringbone/io_buffer_support"
 require_relative "herringbone/codecs/snappy"
 require_relative "herringbone/codecs/lz4"
+require_relative "herringbone/codecs/lzo"
 require_relative "herringbone/thrift"
 require_relative "herringbone/format"
 require_relative "herringbone/encodings/rle"
@@ -32,10 +58,13 @@ require_relative "herringbone/active_record"
 require_relative "herringbone/reader"
 require_relative "herringbone/byte_values"
 require_relative "herringbone/writer"
+require_relative "herringbone/inferring_writer"
+require_relative "herringbone/simple_writer"
 require_relative "herringbone/xxhash"
 require_relative "herringbone/bloom_filter"
 require_relative "herringbone/inspector"
 require_relative "herringbone/visualizer"
+require_relative "herringbone/redaction"
 
 module Herringbone
   module_function
@@ -43,10 +72,13 @@ module Herringbone
   # Writes +records+ to +io+ (any IO responding to #write; Herringbone never opens files by path)
   # and returns the number of rows written. +records+ is an Enumerable of rows, or an ActiveRecord
   # model or relation, which is read with find_each. Without +schema+, the schema comes from the
-  # model's columns (Schema.from_active_record) or is inferred from the first rows (Schema.infer).
-  # Other options go to Writer.
+  # model's columns (Schema.from_active_record) or is inferred from the first rows (Schema.infer);
+  # fields declared in the block replace inferred ones. +records+ is iterated once, so a source
+  # that can only be read once (a cursor, a lazy Enumerator over an IO) works too. Other options go
+  # to Writer.
   #
   #   File.open("orders.parquet", "wb") { |f| Herringbone.write(f, Order.where(created_at: 1.year.ago..)) }
+  #   Herringbone.write(io, events.lazy.map(&:to_h)) { |s| s.json :payload }
   #
   # @param io [IO, #write] destination; written sequentially, never closed
   # @param records [Enumerable<Hash, Array, Object>, Class, #find_each] rows (Hashes, Arrays in schema
@@ -66,27 +98,81 @@ module Herringbone
   # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
   # @option options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
   #   columns to write split block bloom filters for, see Writer
+  # @yield [s] optional, declares fields that replace inferred ones (see Schema.infer); ignored when
+  #   the schema is not inferred
+  # @yieldparam s [Schema::Builder] the builder to declare fields on
+  # @yieldreturn [void]
   # @return [Integer] number of rows written
   # @raise [ArgumentError] when the schema has to be inferred and +records+ is empty or holds Array
-  #   rows, or an option is invalid
+  #   rows, an option is invalid, or the block takes no parameter
   # @raise [EncodeError] when a row does not fit the schema
-  def write(io, records, schema: nil, **options)
+  # @raise [SchemaMismatch] when a row does not fit the inferred schema; the file is left unfinished
+  def write(io, records, schema: nil, **options, &overrides)
+    Schema::Builder.check_block!(overrides, "Herringbone.write(io, rows) { |s| s.json :payload }")
     model = if records.respond_to?(:klass) then records.klass
     elsif records.respond_to?(:columns) && records.respond_to?(:find_each) then records
     end
-    schema ||= model ? Schema.from_active_record(model) : Schema.infer(records)
-    Writer.open(io, schema, **options) do |writer|
+    schema ||= Schema.from_active_record(model) if model
+    writer = if schema
+      Writer.new(io, schema, **options)
+    else
+      fix = "Herringbone.write(io, rows) { |s| s.%s }"
+      InferringWriter.new(io, fix: fix, **options) { |sample| Schema.infer(sample, &overrides) }
+    end
+    begin
       if records.respond_to?(:find_each)
         records.find_each { |record| writer << record }
       else
         records.each { |record| writer << record }
       end
-      writer.rows_written
+    rescue Exception # rubocop:disable Lint/RescueException -- also abort on Interrupt
+      writer.abort
+      raise
+    end
+    writer.close
+    writer.rows_written
+  end
+
+  # Rewrites +input+ into +output+ with rows removed or values replaced, building the Redaction from
+  # the block (or taking +redaction+). A shortcut for Redaction#apply.
+  #
+  #   Herringbone.redact(input, output) do |r|
+  #     r.where(user_id: 42).delete
+  #     r.replace(:email) { |email| email && OpenSSL::HMAC.hexdigest("SHA256", KEY, email) }
+  #   end
+  #
+  # @param input [IO, StringIO] the Parquet file, read with #seek and #read; not closed
+  # @param output [IO, #write] destination, written sequentially; not closed
+  # @param redaction [Redaction, nil] the redaction to apply, instead of a block
+  # @param writer_options [Hash{Symbol => Object}] Writer options for re-encoded column chunks, and
+  #   +metadata:+ to replace the footer key/value metadata, see Redaction#apply
+  # @option writer_options [Symbol] :compression (codec of each source chunk) codec for re-encoded chunks
+  # @option writer_options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
+  #   columns whose re-encoded chunks get a bloom filter, besides those whose source chunk had one
+  # @option writer_options [Hash{String => String}] :metadata (the input's) footer key/value metadata
+  # @option writer_options [Integer] :page_bytes (1MB) approximate uncompressed data page size
+  # @option writer_options [Integer] :page_rows (20_000) maximum rows per data page
+  # @option writer_options [Integer] :data_page_version (1) 1 or 2
+  # @option writer_options [Boolean, Array<String>] :dictionary (true) see Writer
+  # @option writer_options [Hash{String => Symbol}] :encodings ({}) see Writer
+  # @yield [r] declares the statements on a new Redaction (see Redaction.new)
+  # @yieldparam r [Redaction] the redaction being built
+  # @yieldreturn [void]
+  # @return [Redaction::Report] what was done
+  # @raise [ArgumentError] when given both or neither of +redaction+ and a block, a block that takes
+  #   no parameter, or when the redaction does not fit the file's schema
+  # @raise [EncodeError] when a replacement value cannot be written to its column
+  def redact(input, output, redaction = nil, **writer_options, &block)
+    raise ArgumentError, "Herringbone.redact takes a Redaction or a block, not both" if redaction && block
+    raise ArgumentError, "Herringbone.redact needs a Redaction or a block" unless redaction || block
+    if block&.arity&.zero?
+      raise ArgumentError, "The block receives the redaction: Herringbone.redact(input, output) { |r| r.where(user_id: 42).delete }"
     end
+    (redaction || Redaction.new(&block)).apply(input, output, **writer_options)
   end
 
   # Compression codecs this process can read and write, e.g. [:none, :snappy, :gzip, :lz4, :lz4_hadoop, :zstd].
-  # :zstd and :brotli are listed when the zstd-ruby / brotli gems can be loaded.
+  # :zstd and :brotli are listed when the zstd-ruby / brotli gems are loaded.
   #
   # @return [Array<Symbol>] codec names accepted by the writer's +compression:+ option
   def codecs

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: herringbone
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 bindir: bin
 cert_chain: []
-date: 2026-10-01 00:00:00.000000000 Z
+date: 2026-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -33,6 +33,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - LICENSE
 - README.md
 - bin/herringbone
@@ -41,12 +42,14 @@ files:
 - lib/herringbone/bloom_filter.rb
 - lib/herringbone/byte_values.rb
 - lib/herringbone/codecs/lz4.rb
+- lib/herringbone/codecs/lzo.rb
 - lib/herringbone/codecs/snappy.rb
 - lib/herringbone/compression.rb
 - lib/herringbone/encodings/delta.rb
 - lib/herringbone/encodings/plain.rb
 - lib/herringbone/encodings/rle.rb
 - lib/herringbone/format.rb
+- lib/herringbone/inferring_writer.rb
 - lib/herringbone/inspector.rb
 - lib/herringbone/io_buffer_support.rb
 - lib/herringbone/reader.rb
@@ -55,7 +58,10 @@ files:
 - lib/herringbone/reader/numo.rb
 - lib/herringbone/reader/page_stream.rb
 - lib/herringbone/reader/scan.rb
+- lib/herringbone/redaction.rb
+- lib/herringbone/redaction/rewriter.rb
 - lib/herringbone/schema.rb
+- lib/herringbone/simple_writer.rb
 - lib/herringbone/thrift.rb
 - lib/herringbone/types.rb
 - lib/herringbone/version.rb