herringbone 0.4.0 → 0.6.0

data/lib/herringbone.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "stringio"
-require "pathname"
 require_relative "herringbone/version"
 
 # Pure-Ruby reader and writer for Apache Parquet files
@@ -38,32 +37,76 @@ module Herringbone
   class SchemaMismatch < EncodeError; end
 
   # The file (or the writer configuration) uses a Parquet feature Herringbone does not implement,
-  # such as encryption or a codec whose library is unavailable
+  # such as a codec whose library is unavailable
   class UnsupportedError < Error; end
+
+  # An encrypted file or column cannot be read: its key was not given, the key or AAD prefix is
+  # wrong, or the encrypted bytes were changed
+  class DecryptionError < Error; end
 end
 
-require_relative "herringbone/io_buffer_support"
-require_relative "herringbone/codecs/snappy"
-require_relative "herringbone/codecs/lz4"
-require_relative "herringbone/codecs/lzo"
-require_relative "herringbone/thrift"
-require_relative "herringbone/format"
-require_relative "herringbone/encodings/rle"
-require_relative "herringbone/encodings/plain"
-require_relative "herringbone/encodings/delta"
-require_relative "herringbone/compression"
-require_relative "herringbone/types"
-require_relative "herringbone/schema"
-require_relative "herringbone/active_record"
-require_relative "herringbone/reader"
-require_relative "herringbone/byte_values"
-require_relative "herringbone/writer"
-require_relative "herringbone/inferring_writer"
-require_relative "herringbone/simple_writer"
-require_relative "herringbone/xxhash"
-require_relative "herringbone/bloom_filter"
-require_relative "herringbone/inspector"
-require_relative "herringbone/visualizer"
+module Herringbone
+  # Everything else loads on first use, so a process that only reads plain files never loads the
+  # writer, the inspector or OpenSSL. Herringbone.eager_load! loads it all (Ractors on Ruby < 3.4
+  # need that: they cannot autoload).
+
+  # Directory holding the files that are autoloaded
+  LIB = File.expand_path("herringbone", __dir__)
+  private_constant :LIB
+
+  autoload :BloomFilter, "#{LIB}/bloom_filter"
+  autoload :ByteValues, "#{LIB}/byte_values"
+  autoload :Compression, "#{LIB}/compression"
+  autoload :DecryptionConfiguration, "#{LIB}/encryption_configuration"
+  autoload :Encryption, "#{LIB}/encryption"
+  autoload :EncryptionConfiguration, "#{LIB}/encryption_configuration"
+  autoload :Format, "#{LIB}/format"
+  autoload :InferringWriter, "#{LIB}/inferring_writer"
+  autoload :Inspector, "#{LIB}/inspector"
+  autoload :IOBufferSupport, "#{LIB}/io_buffer_support"
+  autoload :Key, "#{LIB}/key"
+  autoload :MissingCodecError, "#{LIB}/compression"
+  autoload :Reader, "#{LIB}/reader"
+  autoload :Redaction, "#{LIB}/redaction"
+  autoload :Schema, "#{LIB}/schema"
+  autoload :SimpleWriter, "#{LIB}/simple_writer"
+  autoload :Thrift, "#{LIB}/thrift"
+  autoload :Types, "#{LIB}/types"
+  autoload :Visualizer, "#{LIB}/visualizer"
+  autoload :Writer, "#{LIB}/writer"
+  autoload :XXHash, "#{LIB}/xxhash"
+
+  module Codecs
+    autoload :LZ4, "#{LIB}/codecs/lz4"
+    autoload :LZO, "#{LIB}/codecs/lzo"
+    autoload :Snappy, "#{LIB}/codecs/snappy"
+  end
+
+  module Encodings
+    autoload :ByteStreamSplit, "#{LIB}/encodings/delta"
+    autoload :Delta, "#{LIB}/encodings/delta"
+    autoload :Plain, "#{LIB}/encodings/plain"
+    autoload :RLE, "#{LIB}/encodings/rle"
+  end
+
+  # Loads every part of Herringbone now instead of on first use. Ractors cannot autoload before
+  # Ruby 3.4, so call this before starting Ractors that use Herringbone there; it also moves the
+  # loading out of the first request in a forking server.
+  #
+  # @return [void]
+  def self.eager_load!
+    seen = {}
+    walk = lambda do |mod|
+      next if seen[mod]
+      seen[mod] = true
+      mod.constants(false).each do |name|
+        value = mod.const_get(name, false)
+        walk.call(value) if value.is_a?(Module) && value.name&.start_with?("Herringbone::")
+      end
+    end
+    walk.call(self)
+  end
+end
 
 module Herringbone
   module_function
@@ -77,7 +120,7 @@ module Herringbone
   # to Writer.
   #
   #   File.open("orders.parquet", "wb") { |f| Herringbone.write(f, Order.where(created_at: 1.year.ago..)) }
-  #   Herringbone.write(io, events.lazy.map(&:to_h)) { json :payload }
+  #   Herringbone.write(io, events.lazy.map(&:to_h)) { |s| s.json :payload }
   #
   # @param io [IO, #write] destination; written sequentially, never closed
   # @param records [Enumerable<Hash, Array, Object>, Class, #find_each] rows (Hashes, Arrays in schema
@@ -85,6 +128,7 @@ module Herringbone
   # @param schema [Schema, nil] schema to write with; derived from +records+ when nil
   # @param options [Hash{Symbol => Object}] passed to Writer.new
   # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+  # @option options [Integer, nil] :compression_level (nil) level for :zstd, :gzip or :brotli
   # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
   # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
   # @option options [Integer] :page_bytes (1MB) approximate uncompressed data page size
@@ -97,14 +141,17 @@ module Herringbone
   # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
   # @option options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
   #   columns to write split block bloom filters for, see Writer
-  # @yield optional block for Schema.infer (Builder DSL), declaring fields that replace inferred ones;
-  #   ignored when the schema is not inferred
+  # @yield [s] optional, declares fields that replace inferred ones (see Schema.infer); ignored when
+  #   the schema is not inferred
+  # @yieldparam s [Schema::Builder] the builder to declare fields on
+  # @yieldreturn [void]
   # @return [Integer] number of rows written
   # @raise [ArgumentError] when the schema has to be inferred and +records+ is empty or holds Array
-  #   rows, or an option is invalid
+  #   rows, an option is invalid, or the block takes no parameter
   # @raise [EncodeError] when a row does not fit the schema
   # @raise [SchemaMismatch] when a row does not fit the inferred schema; the file is left unfinished
   def write(io, records, schema: nil, **options, &overrides)
+    Schema::Builder.check_block!(overrides, "Herringbone.write(io, rows) { |s| s.json :payload }")
     model = if records.respond_to?(:klass) then records.klass
     elsif records.respond_to?(:columns) && records.respond_to?(:find_each) then records
     end
@@ -112,7 +159,7 @@ module Herringbone
     writer = if schema
       Writer.new(io, schema, **options)
     else
-      fix = "Herringbone.write(io, rows) { %s }"
+      fix = "Herringbone.write(io, rows) { |s| s.%s }"
       InferringWriter.new(io, fix: fix, **options) { |sample| Schema.infer(sample, &overrides) }
     end
     begin
@@ -129,6 +176,45 @@ module Herringbone
     writer.rows_written
   end
 
+  # Rewrites +input+ into +output+ with rows removed or values replaced, building the Redaction from
+  # the block (or taking +redaction+). A shortcut for Redaction#apply.
+  #
+  #   Herringbone.redact(input, output) do |r|
+  #     r.where(user_id: 42).delete
+  #     r.replace(:email) { |email| email && OpenSSL::HMAC.hexdigest("SHA256", KEY, email) }
+  #   end
+  #
+  # @param input [IO, StringIO] the Parquet file, read with #seek and #read; not closed
+  # @param output [IO, #write] destination, written sequentially; not closed
+  # @param redaction [Redaction, nil] the redaction to apply, instead of a block
+  # @param writer_options [Hash{Symbol => Object}] Writer options for re-encoded column chunks, and
+  #   +metadata:+ to replace the footer key/value metadata, see Redaction#apply
+  # @option writer_options [Symbol] :compression (codec of each source chunk) codec for re-encoded chunks
+  # @option writer_options [Integer, nil] :compression_level (nil) level for that codec, see Writer
+  # @option writer_options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
+  #   columns whose re-encoded chunks get a bloom filter, besides those whose source chunk had one
+  # @option writer_options [Hash{String => String}] :metadata (the input's) footer key/value metadata
+  # @option writer_options [Integer] :page_bytes (1MB) approximate uncompressed data page size
+  # @option writer_options [Integer] :page_rows (20_000) maximum rows per data page
+  # @option writer_options [Integer] :data_page_version (1) 1 or 2
+  # @option writer_options [Boolean, Array<String>] :dictionary (true) see Writer
+  # @option writer_options [Hash{String => Symbol}] :encodings ({}) see Writer
+  # @yield [r] declares the statements on a new Redaction (see Redaction.new)
+  # @yieldparam r [Redaction] the redaction being built
+  # @yieldreturn [void]
+  # @return [Redaction::Report] what was done
+  # @raise [ArgumentError] when given both or neither of +redaction+ and a block, a block that takes
+  #   no parameter, or when the redaction does not fit the file's schema
+  # @raise [EncodeError] when a replacement value cannot be written to its column
+  def redact(input, output, redaction = nil, **writer_options, &block)
+    raise ArgumentError, "Herringbone.redact takes a Redaction or a block, not both" if redaction && block
+    raise ArgumentError, "Herringbone.redact needs a Redaction or a block" unless redaction || block
+    if block&.arity&.zero?
+      raise ArgumentError, "The block receives the redaction: Herringbone.redact(input, output) { |r| r.where(user_id: 42).delete }"
+    end
+    (redaction || Redaction.new(&block)).apply(input, output, **writer_options)
+  end
+
   # Compression codecs this process can read and write, e.g. [:none, :snappy, :gzip, :lz4, :lz4_hadoop, :zstd].
   # :zstd and :brotli are listed when the zstd-ruby / brotli gems are loaded.
   #

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: herringbone
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 bindir: bin
 cert_chain: []
-date: 2026-10-02 00:00:00.000000000 Z
+date: 2026-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -35,6 +35,7 @@ extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE
+- MANUAL.md
 - README.md
 - bin/herringbone
 - lib/herringbone.rb
@@ -48,16 +49,22 @@ files:
 - lib/herringbone/encodings/delta.rb
 - lib/herringbone/encodings/plain.rb
 - lib/herringbone/encodings/rle.rb
+- lib/herringbone/encryption.rb
+- lib/herringbone/encryption_configuration.rb
 - lib/herringbone/format.rb
 - lib/herringbone/inferring_writer.rb
 - lib/herringbone/inspector.rb
 - lib/herringbone/io_buffer_support.rb
+- lib/herringbone/key.rb
 - lib/herringbone/reader.rb
+- lib/herringbone/reader/bloom_filters.rb
 - lib/herringbone/reader/column_chunk_reader.rb
 - lib/herringbone/reader/column_cursor.rb
 - lib/herringbone/reader/numo.rb
 - lib/herringbone/reader/page_stream.rb
 - lib/herringbone/reader/scan.rb
+- lib/herringbone/redaction.rb
+- lib/herringbone/redaction/rewriter.rb
 - lib/herringbone/schema.rb
 - lib/herringbone/simple_writer.rb
 - lib/herringbone/thrift.rb