herringbone 0.6.1 → 0.7.0

data/lib/herringbone/writer.rb

@@ -76,6 +76,8 @@ module Herringbone
     MAX_DICTIONARY_BYTES = 1024 * 1024
     # The row group byte size is first estimated after this many rows, then after every row group
     ESTIMATE_AFTER_ROWS = 1000
+    # Options .open_for_copies refuses, since the row groups it writes keep their boundaries
+    ROW_GROUP_OPTIONS = %i[row_group_bytes row_group_rows].freeze
 
     # @return [Schema] schema the rows are written with
     attr_reader :schema
@@ -121,6 +123,29 @@ module Herringbone
       result
     end
 
+    # Internal (used by Redaction and Combiner): .open for a file whose row groups are all written
+    # with #write_row_group, one for each row group of another file, so they keep their boundaries
+    #
+    # @param io [IO, #write] destination
+    # @param schema [Schema] schema of the rows
+    # @param refusal [String] why +row_group_bytes:+ and +row_group_rows:+ cannot be given, e.g.
+    #   "a redaction keeps the row groups of the input"
+    # @param options [Hash{Symbol => Object}] see .open, without the row group options
+    # @option options [Symbol] :compression (:snappy) codec of the chunks encoded again
+    # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
+    # @option options [EncryptionConfiguration, Hash{Symbol => Object}, nil] :encryption (nil) see #initialize
+    # @yield [writer] the open writer
+    # @yieldparam writer [Writer] writer to write row groups to
+    # @yieldreturn [Object] returned by open_for_copies
+    # @return [Object] the block's value
+    # @raise [ArgumentError] for +row_group_bytes:+ / +row_group_rows:+, or an invalid option
+    # @api private
+    def self.open_for_copies(io, schema, refusal, **options, &block)
+      bad = options.keys & ROW_GROUP_OPTIONS
+      raise ArgumentError, "#{bad.join(", ")}: #{refusal}" unless bad.empty?
+      self.open(io, schema, row_group_bytes: 1 << 62, **options, &block)
+    end
+
     # Validates the options and writes the leading magic bytes to +io+.
     #
     # @param io [IO, #write] destination; switched to binmode when it supports that
@@ -269,11 +294,12 @@ module Herringbone
     #   @return [Format::OffsetIndex, nil] the chunk's OffsetIndex, with source file offsets
     # @!attribute bloom_filter
     #   @return [String, nil] the chunk's encoded bloom filter, header included
+    # @api private
     CopiedChunk = Struct.new(:chunk, :start, :bytes, :column_index, :offset_index, :bloom_filter)
 
-    # Internal (used by Redaction): writes a row group of +num_rows+ rows from the buffered values,
-    # except for the columns in +copies+, whose chunks are copied byte for byte from another file
-    # with their offsets rebased. Starts a new row group afterwards.
+    # Internal (used by Redaction and Combiner): writes a row group of +num_rows+ rows from the
+    # buffered values, except for the columns in +copies+, whose chunks are copied byte for byte
+    # from another file with their offsets rebased. Starts a new row group afterwards.
     #
     # @param num_rows [Integer] rows in the row group
     # @param copies [Hash{Integer => CopiedChunk}] column index => chunk to copy instead of encoding
@@ -285,6 +311,7 @@ module Herringbone
     # @param sorting_columns [Array<Format::SortingColumn>, nil] stored on the RowGroup as given
     # @return [void]
     # @raise [Error] when the writer is closed
+    # @api private
     def write_row_group(num_rows, copies: {}, codecs: {}, bloom_filters: {}, sorting_columns: nil)
       raise Error, "Writer is closed" if @closed
       if @encryption && (encrypted = copies.keys.find { |i| @encryption.encrypted?(i) })
@@ -293,7 +320,7 @@ module Herringbone
       start = @pos
       chunks = @schema.columns.map do |col|
         if (copy = copies[col.index])
-          copy_column_chunk(copy)
+          copy_column_chunk(col, copy)
         else
           bloom = @bloom_filters[col.dotted_path]
           bloom ||= bloom_filter_settings({}) if bloom_filters[col.index] && BloomFilter::TYPES.include?(col.type)
@@ -314,21 +341,23 @@ module Herringbone
       reset_buffers
     end
 
-    # Internal (used by Redaction): whether the column is encrypted in this file
+    # Internal (used by Redaction and Combiner): whether the column is encrypted in this file
     #
     # @param index [Integer] leaf column index
     # @return [Boolean]
+    # @api private
     def encrypted_column?(index) = @encryption&.encrypted?(index) || false
 
-    # Internal (used by Redaction): shreds one value per row of a top-level field into the buffers,
-    # for #write_row_group. Other fields are left as they are, so the caller decides which columns
-    # are encoded and which are copied.
+    # Internal (used by Redaction and Combiner): shreds one value per row of a top-level field into
+    # the buffers, for #write_row_group. Other fields are left as they are, so the caller decides
+    # which columns are encoded and which are copied.
     #
     # @param name [String] top-level field name
     # @param values [Array] the field's value for each row
     # @return [void]
     # @raise [ArgumentError] when the schema has no such field
     # @raise [EncodeError] when a value cannot be written to the field
+    # @api private
     def buffer_field(name, values)
       field = @schema.field(name) or raise ArgumentError, "No such field #{name.inspect}"
       values.each_with_index do |value, i|
@@ -414,6 +443,7 @@ module Herringbone
     #   @return [String, Array<Integer>, nil] repetition levels, like +defs+; nil for non-repeated columns
     # @!attribute values
     #   @return [Array, ByteValues] non-null physical values
+    # @api private
     ColumnBuffer = Struct.new(:defs, :reps, :values)
 
     # Starts a new row group: fresh buffers per column, and the per-field write plan
@@ -769,6 +799,7 @@ module Herringbone
     #   @return [Integer] non-null values in the page
     # @!attribute range
     #   @return [Array(Object, Object), nil] [min, max] physical values, nil when there are none
+    # @api private
     PageInfo = Struct.new(:offset, :size, :first_row, :nulls, :non_null, :range)
 
     # @param pages [Array<PageInfo>] pages of a column chunk
@@ -781,14 +812,17 @@ module Herringbone
 
     # Writes a chunk copied from another file. The pages, the ColumnIndex and the bloom filter
     # hold no file offsets and go out as they are; the ColumnMetaData and the OffsetIndex do, so
-    # those are rebased onto where the chunk lands in this file.
+    # those are rebased onto where the chunk lands in this file. The path is this file's, which
+    # differs when the source file names list elements another way ("list.item", "array").
     #
+    # @param col [Schema::Column] the column the chunk is copied into
     # @param copy [CopiedChunk] the chunk to copy
     # @return [Format::ColumnChunk] chunk with its ColumnMetaData, for the row group
-    def copy_column_chunk(copy)
+    def copy_column_chunk(col, copy)
       start = @pos
       shift = start - copy.start
       meta = Format::ColumnMetaData.decode(copy.chunk.meta_data.encode).first
+      meta.path_in_schema = col.path
       meta.data_page_offset += shift
       dict = meta.dictionary_page_offset
       # Some writers store 0 when there is no dictionary page
@@ -1070,7 +1104,7 @@ module Herringbone
     # keeps in their source codec use that codec's default
     #
     # @param codec [Integer] codec id
-    # @param data [String] bytes to compress
+    # @param data [String, Array<String>] bytes to compress, or the parts of them
     # @return [String] compressed bytes
     def compress(codec, data)
       Compression.compress(codec, data, (codec == @codec) ? @compression_level : nil)
@@ -1080,28 +1114,30 @@ module Herringbone
     # stored, encrypted.
     #
     # @param header [Format::PageHeader] header; sizes and CRC32 are filled in here
-    # @param body [String] uncompressed page body
-    # @param compressed [String, nil] bytes to write as the page body, when already prepared (v2 data
-    #   pages, whose levels stay uncompressed); +body+ is compressed otherwise
+    # @param body [String, Array<String>] uncompressed page body, or its parts
+    # @param compressed [String, Array<String>, nil] bytes to write as the page body, or their
+    #   parts, when already prepared (v2 data pages, whose levels stay uncompressed); +body+ is
+    #   compressed otherwise
     # @param codec [Integer] codec id to compress +body+ with
     # @param crypto [Encryption::ModuleCrypto, nil] encryption of the column's modules
     # @param ordinal [Integer, nil] data page ordinal within the chunk; nil for a dictionary page
     # @return [Integer]
     def write_page(header, body, compressed = nil, codec: @codec, crypto: nil, ordinal: nil)
-      compressed ||= compress(codec, body)
-      header.uncompressed_page_size ||= body.bytesize
+      parts = Array(compressed || compress(codec, body))
+      header.uncompressed_page_size ||= Array(body).sum(&:bytesize)
       dictionary = header.type == Format::PageType::DICTIONARY_PAGE
       if crypto
-        compressed = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE, compressed, ordinal)
+        parts = [crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE, parts.join, ordinal)]
       end
-      header.compressed_page_size = compressed.bytesize
-      header.crc = Zlib.crc32(compressed).then { |c| (c >= 0x8000_0000) ? c - 0x1_0000_0000 : c }
+      header.compressed_page_size = parts.sum(&:bytesize)
+      crc = parts.inject(0) { |c, part| Zlib.crc32(part, c) }
+      header.crc = (crc >= 0x8000_0000) ? crc - 0x1_0000_0000 : crc
       encoded = header.encode
       if crypto
         encoded = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE_HEADER : Encryption::DATA_PAGE_HEADER, encoded, ordinal)
       end
       write_raw(encoded)
-      write_raw(compressed)
+      parts.each { |part| write_raw(part) unless part.empty? }
       encoded.bytesize + header.uncompressed_page_size
     end
 
@@ -1117,7 +1153,7 @@ module Herringbone
     #   modules and the page's ordinal, for an encrypted column
     # @return [Integer] uncompressed size including the header
     def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding, codec, page_crypto)
-      body = String.new(encoding: Encoding::BINARY)
+      body = []
       body << [rep_bytes.bytesize].pack("V") << rep_bytes unless rep_bytes.empty?
       body << [def_bytes.bytesize].pack("V") << def_bytes unless def_bytes.empty?
       body << encoded
@@ -1156,7 +1192,7 @@ module Herringbone
           is_compressed: codec != Format::Codec::UNCOMPRESSED
         )
       )
-      write_page(header, "".b, rep_bytes + def_bytes + compressed, codec: codec, crypto: page_crypto&.first,
+      write_page(header, "".b, [rep_bytes, def_bytes, compressed], codec: codec, crypto: page_crypto&.first,
         ordinal: page_crypto&.last)
     end
 

data/lib/herringbone/xxhash.rb

@@ -14,6 +14,8 @@ module Herringbone
   # allocations trigger so many garbage collections that hashing becomes up to 10x slower. The
   # arithmetic is written once with the small code generators below, whose output is inlined into
   # the hashing methods: no method calls or allocations in the hot paths.
+  #
+  # @api private
   module XXHash
     # 64-bit mask
     M = 0xFFFF_FFFF_FFFF_FFFF

data/lib/herringbone.rb

@@ -40,6 +40,34 @@ module Herringbone
   # such as a codec whose library is unavailable
   class UnsupportedError < Error; end
 
+  # Schemas that do not fit together: two schemas that cannot be united or intersected
+  # (Schema#union, Schema#intersect), or inputs of Herringbone.combine that do not fit the output
+  # schema or each other. The message lists every field that does not fit, one per line, e.g.
+  #
+  #   Cannot unite the schemas, 2 fields do not fit:
+  #     price: int64 vs double (int64 does not fit a double exactly)
+  #     address.zip: int32 vs string (no common type)
+  class IncompatibleSchema < Error
+    # A field that does not fit: its dotted path, how each side declares it, why the two do not
+    # fit, and for Herringbone.combine the position of the input it was found in
+    Conflict = Struct.new(:path, :left, :right, :reason, :input) do
+      # @return [String] e.g. "address.zip: int32 vs string (no common type)"
+      def to_s = "#{path.empty? ? "(top level)" : path}: #{left} vs #{right} (#{reason})"
+    end
+
+    # @return [Array<Conflict>] every field that does not fit, in schema (and input) order
+    attr_reader :conflicts
+
+    # @param conflicts [Array<Conflict>] the fields that do not fit
+    # @param operation [String, nil] what was attempted, "unite" or "intersect", for the message
+    # @param message [String, nil] the message, instead of one built from +operation+
+    def initialize(conflicts, operation: nil, message: nil)
+      @conflicts = conflicts
+      count = (conflicts.size == 1) ? "1 field does" : "#{conflicts.size} fields do"
+      super(message || "Cannot #{operation} the schemas, #{count} not fit:\n#{conflicts.map { |c| "  #{c}" }.join("\n")}")
+    end
+  end
+
   # An encrypted file or column cannot be read: its key was not given, the key or AAD prefix is
   # wrong, or the encrypted bytes were changed
   class DecryptionError < Error; end
@@ -56,6 +84,7 @@ module Herringbone
 
   autoload :BloomFilter, "#{LIB}/bloom_filter"
   autoload :ByteValues, "#{LIB}/byte_values"
+  autoload :Combiner, "#{LIB}/combiner"
   autoload :Compression, "#{LIB}/compression"
   autoload :DecryptionConfiguration, "#{LIB}/encryption_configuration"
   autoload :Encryption, "#{LIB}/encryption"
@@ -68,6 +97,7 @@ module Herringbone
   autoload :MissingCodecError, "#{LIB}/compression"
   autoload :Reader, "#{LIB}/reader"
   autoload :Redaction, "#{LIB}/redaction"
+  autoload :RestrictedReadableIO, "#{LIB}/restricted_readable_io"
   autoload :Schema, "#{LIB}/schema"
   autoload :SimpleWriter, "#{LIB}/simple_writer"
   autoload :Thrift, "#{LIB}/thrift"
@@ -176,16 +206,17 @@ module Herringbone
     writer.rows_written
   end
 
-  # Rewrites +input+ into +output+ with rows removed or values replaced, building the Redaction from
-  # the block (or taking +redaction+). A shortcut for Redaction#apply.
+  # Rewrites +io_or_reader+ into +output_io+ with rows removed or values replaced, building the
+  # Redaction from the block (or taking +redaction+). A shortcut for Redaction#apply.
   #
-  #   Herringbone.redact(input, output) do |r|
+  #   Herringbone.redact(io, output_io) do |r|
   #     r.where(user_id: 42).delete
   #     r.replace(:email) { |email| email && OpenSSL::HMAC.hexdigest("SHA256", KEY, email) }
   #   end
   #
-  # @param input [IO, StringIO] the Parquet file, read with #seek and #read; not closed
-  # @param output [IO, #write] destination, written sequentially; not closed
+  # @param io_or_reader [IO, StringIO, Reader] the Parquet file, read with #seek and #read, or a
+  #   Reader of it, whose IO and decryption are used; not closed
+  # @param output_io [IO, #write] destination, written sequentially; not closed
   # @param redaction [Redaction, nil] the redaction to apply, instead of a block
   # @param writer_options [Hash{Symbol => Object}] Writer options for re-encoded column chunks, and
   #   +metadata:+ to replace the footer key/value metadata, see Redaction#apply
@@ -204,15 +235,71 @@ module Herringbone
   # @yieldreturn [void]
   # @return [Redaction::Report] what was done
   # @raise [ArgumentError] when given both or neither of +redaction+ and a block, a block that takes
-  #   no parameter, or when the redaction does not fit the file's schema
+  #   no parameter, a Reader with +decryption:+, or when the redaction does not fit the file's schema
   # @raise [EncodeError] when a replacement value cannot be written to its column
-  def redact(input, output, redaction = nil, **writer_options, &block)
+  def redact(io_or_reader, output_io, redaction = nil, **writer_options, &block)
     raise ArgumentError, "Herringbone.redact takes a Redaction or a block, not both" if redaction && block
     raise ArgumentError, "Herringbone.redact needs a Redaction or a block" unless redaction || block
     if block&.arity&.zero?
-      raise ArgumentError, "The block receives the redaction: Herringbone.redact(input, output) { |r| r.where(user_id: 42).delete }"
+      raise ArgumentError, "The block receives the redaction: Herringbone.redact(io, output_io) { |r| r.where(user_id: 42).delete }"
     end
-    (redaction || Redaction.new(&block)).apply(input, output, **writer_options)
+    (redaction || Redaction.new(&block)).apply(io_or_reader, output_io, **writer_options)
+  end
+
+  # Concatenates Parquet files into +output_io+: every row group of every input, in order. A
+  # shortcut for Combiner. Column chunks are copied byte for byte (only offsets are rebased)
+  # wherever the output stores a column's values as the input does, so files with the same schema
+  # are combined without decoding anything.
+  #
+  #   Herringbone.combine([jan, feb, mar], output_io)                      # the same schema
+  #   Herringbone.combine([old, new], output_io, schema: :union)           # all fields, nulls in the gaps
+  #   Herringbone.combine([a, b], output_io, schema: :intersect)           # the fields all inputs have
+  #
+  # Without +schema:+ every input must have the schema of the first (see Schema#==). With
+  # +schema: :union+ the output has the fields of all inputs, widened as Schema#union widens them,
+  # and fields an input lacks are written as nulls. With +schema: :intersect+ it has the fields all
+  # inputs have, and the others are dropped. A Schema given as +schema:+ must hold every input: each
+  # must have the same fields or fewer, in the same or narrower types. Every input is checked
+  # before anything is written, and IncompatibleSchema lists all that do not fit.
+  #
+  # The output is encrypted like the encrypted inputs, if any (plaintext inputs included), unless
+  # +encryption:+ says otherwise; +encryption: false+ writes it in plaintext. Inputs encrypted
+  # differently need +encryption:+. Herringbone never opens files by path: pass open IOs, which
+  # are read twice (footers first, then chunks), so keep them open until +combine+ returns and
+  # mind the limit on open files when combining very many of them.
+  #
+  # @param ios_or_readers [Enumerable<IO, StringIO, Reader>] the Parquet files: IOs read with #seek
+  #   and #read, or Readers, which bring their own decryption; enumerated once, none is closed
+  # @param output_io [IO, #write] destination, written sequentially; not closed
+  # @param schema [Schema, Symbol, nil] +:union+, +:intersect+, a Schema every input fits, or nil
+  #   to require the inputs to have the same schema
+  # @param decryption [DecryptionConfiguration, Hash{Symbol => Object}, Array, #call, nil] keys of
+  #   encrypted inputs given as IOs, see Reader.new
+  # @param writer_options [Hash{Symbol => Object}] Writer options for chunks encoded again (and
+  #   written nulls), +metadata:+ and +encryption:+
+  # @option writer_options [Symbol] :compression (codec of each source chunk) codec for chunks
+  #   encoded again
+  # @option writer_options [Integer, nil] :compression_level (nil) level for that codec, see Writer
+  # @option writer_options [Hash{String => String}] :metadata (the first input's) footer key/value metadata
+  # @option writer_options [EncryptionConfiguration, Hash{Symbol => Object}, Key, String, false] :encryption
+  #   (encrypted like the encrypted inputs) see Writer; false for a plaintext output
+  # @option writer_options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
+  #   columns whose encoded chunks get a bloom filter, besides those whose source chunk had one
+  # @option writer_options [Integer] :page_bytes (1MB) approximate uncompressed data page size
+  # @option writer_options [Integer] :page_rows (20_000) maximum rows per data page
+  # @option writer_options [Integer] :data_page_version (1) 1 or 2
+  # @option writer_options [Boolean, Array<String>] :dictionary (true) see Writer
+  # @option writer_options [Hash{String => Symbol}] :encodings ({}) see Writer
+  # @return [Combiner::Report] rows written, how many row groups were copied or rewritten, and per
+  #   input which fields were filled with nulls, widened or dropped
+  # @raise [IncompatibleSchema] listing every input (and field) that does not fit the others or
+  #   +schema:+
+  # @raise [ArgumentError] when there are no inputs, an input is neither an IO nor a Reader, the
+  #   encrypted inputs are encrypted differently and +encryption:+ is not given, or for
+  #   +row_group_bytes:+ / +row_group_rows:+
+  # @raise [DecryptionError] when an input is encrypted and a key it needs was not given
+  def combine(ios_or_readers, output_io, schema: nil, decryption: nil, **writer_options)
+    Combiner.new(ios_or_readers, schema: schema, decryption: decryption).apply(output_io, **writer_options)
   end
 
   # Compression codecs this process can read and write, e.g. [:none, :snappy, :gzip, :lz4, :lz4_hadoop, :zstd].

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: herringbone
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 bindir: bin
 cert_chain: []
-date: 2026-10-04 00:00:00.000000000 Z
+date: 2026-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -45,6 +45,7 @@ files:
 - lib/herringbone/codecs/lz4.rb
 - lib/herringbone/codecs/lzo.rb
 - lib/herringbone/codecs/snappy.rb
+- lib/herringbone/combiner.rb
 - lib/herringbone/compression.rb
 - lib/herringbone/encodings/delta.rb
 - lib/herringbone/encodings/plain.rb
@@ -58,6 +59,7 @@ files:
 - lib/herringbone/key.rb
 - lib/herringbone/reader.rb
 - lib/herringbone/reader/bloom_filters.rb
+- lib/herringbone/reader/chunk_copier.rb
 - lib/herringbone/reader/column_chunk_reader.rb
 - lib/herringbone/reader/column_cursor.rb
 - lib/herringbone/reader/numo.rb
@@ -65,7 +67,9 @@ files:
 - lib/herringbone/reader/scan.rb
 - lib/herringbone/redaction.rb
 - lib/herringbone/redaction/rewriter.rb
+- lib/herringbone/restricted_readable_io.rb
 - lib/herringbone/schema.rb
+- lib/herringbone/schema/merge.rb
 - lib/herringbone/simple_writer.rb
 - lib/herringbone/thrift.rb
 - lib/herringbone/types.rb