herringbone 0.5.0 → 0.6.1

data/lib/herringbone.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "stringio"
-require "pathname"
 require_relative "herringbone/version"
 
 # Pure-Ruby reader and writer for Apache Parquet files
@@ -38,33 +37,76 @@ module Herringbone
   class SchemaMismatch < EncodeError; end
 
   # The file (or the writer configuration) uses a Parquet feature Herringbone does not implement,
-  # such as encryption or a codec whose library is unavailable
+  # such as a codec whose library is unavailable
   class UnsupportedError < Error; end
+
+  # An encrypted file or column cannot be read: its key was not given, the key or AAD prefix is
+  # wrong, or the encrypted bytes were changed
+  class DecryptionError < Error; end
 end
 
-require_relative "herringbone/io_buffer_support"
-require_relative "herringbone/codecs/snappy"
-require_relative "herringbone/codecs/lz4"
-require_relative "herringbone/codecs/lzo"
-require_relative "herringbone/thrift"
-require_relative "herringbone/format"
-require_relative "herringbone/encodings/rle"
-require_relative "herringbone/encodings/plain"
-require_relative "herringbone/encodings/delta"
-require_relative "herringbone/compression"
-require_relative "herringbone/types"
-require_relative "herringbone/schema"
-require_relative "herringbone/active_record"
-require_relative "herringbone/reader"
-require_relative "herringbone/byte_values"
-require_relative "herringbone/writer"
-require_relative "herringbone/inferring_writer"
-require_relative "herringbone/simple_writer"
-require_relative "herringbone/xxhash"
-require_relative "herringbone/bloom_filter"
-require_relative "herringbone/inspector"
-require_relative "herringbone/visualizer"
-require_relative "herringbone/redaction"
+module Herringbone
+  # Everything else loads on first use, so a process that only reads plain files never loads the
+  # writer, the inspector or OpenSSL. Herringbone.eager_load! loads it all (Ractors on Ruby < 3.4
+  # need that: they cannot autoload).
+
+  # Directory holding the files that are autoloaded
+  LIB = File.expand_path("herringbone", __dir__)
+  private_constant :LIB
+
+  autoload :BloomFilter, "#{LIB}/bloom_filter"
+  autoload :ByteValues, "#{LIB}/byte_values"
+  autoload :Compression, "#{LIB}/compression"
+  autoload :DecryptionConfiguration, "#{LIB}/encryption_configuration"
+  autoload :Encryption, "#{LIB}/encryption"
+  autoload :EncryptionConfiguration, "#{LIB}/encryption_configuration"
+  autoload :Format, "#{LIB}/format"
+  autoload :InferringWriter, "#{LIB}/inferring_writer"
+  autoload :Inspector, "#{LIB}/inspector"
+  autoload :IOBufferSupport, "#{LIB}/io_buffer_support"
+  autoload :Key, "#{LIB}/key"
+  autoload :MissingCodecError, "#{LIB}/compression"
+  autoload :Reader, "#{LIB}/reader"
+  autoload :Redaction, "#{LIB}/redaction"
+  autoload :Schema, "#{LIB}/schema"
+  autoload :SimpleWriter, "#{LIB}/simple_writer"
+  autoload :Thrift, "#{LIB}/thrift"
+  autoload :Types, "#{LIB}/types"
+  autoload :Visualizer, "#{LIB}/visualizer"
+  autoload :Writer, "#{LIB}/writer"
+  autoload :XXHash, "#{LIB}/xxhash"
+
+  module Codecs
+    autoload :LZ4, "#{LIB}/codecs/lz4"
+    autoload :LZO, "#{LIB}/codecs/lzo"
+    autoload :Snappy, "#{LIB}/codecs/snappy"
+  end
+
+  module Encodings
+    autoload :ByteStreamSplit, "#{LIB}/encodings/delta"
+    autoload :Delta, "#{LIB}/encodings/delta"
+    autoload :Plain, "#{LIB}/encodings/plain"
+    autoload :RLE, "#{LIB}/encodings/rle"
+  end
+
+  # Loads every part of Herringbone now instead of on first use. Ractors cannot autoload before
+  # Ruby 3.4, so call this before starting Ractors that use Herringbone there; it also moves the
+  # loading out of the first request in a forking server.
+  #
+  # @return [void]
+  def self.eager_load!
+    seen = {}
+    walk = lambda do |mod|
+      next if seen[mod]
+      seen[mod] = true
+      mod.constants(false).each do |name|
+        value = mod.const_get(name, false)
+        walk.call(value) if value.is_a?(Module) && value.name&.start_with?("Herringbone::")
+      end
+    end
+    walk.call(self)
+  end
+end
 
 module Herringbone
   module_function
@@ -86,6 +128,7 @@ module Herringbone
   # @param schema [Schema, nil] schema to write with; derived from +records+ when nil
   # @param options [Hash{Symbol => Object}] passed to Writer.new
   # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+  # @option options [Integer, nil] :compression_level (nil) level for :zstd, :gzip or :brotli
   # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
   # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
   # @option options [Integer] :page_bytes (1MB) approximate uncompressed data page size
@@ -147,6 +190,7 @@ module Herringbone
   # @param writer_options [Hash{Symbol => Object}] Writer options for re-encoded column chunks, and
   #   +metadata:+ to replace the footer key/value metadata, see Redaction#apply
   # @option writer_options [Symbol] :compression (codec of each source chunk) codec for re-encoded chunks
+  # @option writer_options [Integer, nil] :compression_level (nil) level for that codec, see Writer
   # @option writer_options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
   #   columns whose re-encoded chunks get a bloom filter, besides those whose source chunk had one
   # @option writer_options [Hash{String => String}] :metadata (the input's) footer key/value metadata

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: herringbone
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Julik Tarkhanov
 bindir: bin
 cert_chain: []
-date: 2026-10-02 00:00:00.000000000 Z
+date: 2026-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -35,6 +35,7 @@ extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE
+- MANUAL.md
 - README.md
 - bin/herringbone
 - lib/herringbone.rb
@@ -48,11 +49,15 @@ files:
 - lib/herringbone/encodings/delta.rb
 - lib/herringbone/encodings/plain.rb
 - lib/herringbone/encodings/rle.rb
+- lib/herringbone/encryption.rb
+- lib/herringbone/encryption_configuration.rb
 - lib/herringbone/format.rb
 - lib/herringbone/inferring_writer.rb
 - lib/herringbone/inspector.rb
 - lib/herringbone/io_buffer_support.rb
+- lib/herringbone/key.rb
 - lib/herringbone/reader.rb
+- lib/herringbone/reader/bloom_filters.rb
 - lib/herringbone/reader/column_chunk_reader.rb
 - lib/herringbone/reader/column_cursor.rb
 - lib/herringbone/reader/numo.rb