herringbone 0.6.1 → 0.8.0

data/lib/herringbone/thrift.rb

@@ -4,6 +4,8 @@ module Herringbone
   # Minimal Thrift Compact Protocol implementation, just enough for Parquet metadata.
   # Structs are described declaratively (see Herringbone::Thrift::Struct) so that
   # both the reader and the writer are driven by the same field tables.
+  #
+  # @api private
   module Thrift
     # Raised on malformed or truncated Thrift data, and on values the writer cannot encode
     class Error < StandardError; end
@@ -47,7 +49,7 @@ module Herringbone
 
     # Values each declared integer type can hold, whatever width it came in on the wire
     INT_RANGES = {
-      i16: -2**15...2**15, i32: -2**31...2**31, i64: -2**63...2**63
+      byte: -2**7...2**7, i16: -2**15...2**15, i32: -2**31...2**31, i64: -2**63...2**63
     }.freeze
 
     # Deepest nesting of structs, lists, sets and maps read or skipped, as in Apache Thrift
@@ -94,7 +96,9 @@ module Herringbone
 
       # @param buf [String] the encoded data (binary)
       # @param pos [Integer] byte offset to start reading at
+      # @raise [Error] when +pos+ is negative
       def initialize(buf, pos = 0)
+        raise Error, "Negative offset #{pos}" if pos.negative?
         @buf = buf
         @pos = pos
         @depth = 0
@@ -197,8 +201,8 @@ module Herringbone
       # Reads one value of wire type +wire+ as declared +type+
       #
       # @param wire [Integer] wire type from the field or list header
-      # @param type [Symbol, Array, Class] declared type; +:string+ makes binary values UTF-8,
-      #   an Array or Struct subclass gives the element type or struct to read
+      # @param type [Symbol, Array, Class] declared type; +:string+ makes binary values UTF-8 and
+      #   +:binary+ makes them BINARY, an Array or Struct subclass gives the element type or struct to read
       # @return [Object] true/false, an Integer (bytes are signed), a Float, a String, an Array
       #   (or nil, see #read_list) or a Struct
       # @raise [Error] on an unsupported wire type, truncated data or an integer out of the range
@@ -217,7 +221,7 @@ module Herringbone
         when T_DOUBLE then read_double
         when T_BINARY
           s = read_binary
-          (type == :string) ? s.force_encoding(Encoding::UTF_8) : s
+          s.force_encoding((type == :string) ? Encoding::UTF_8 : Encoding::BINARY)
         when T_LIST, T_SET then read_list(type)
         when T_STRUCT then read_struct(type)
         else
@@ -407,12 +411,15 @@ module Herringbone
       # @param type [Symbol, Array, Class] declared type (see WIRE_TYPES)
       # @param value [Object] Integer, Float, String, Array or Struct matching +type+
       # @return [void]
+      # @raise [Error] when an integer type gets a value that is not an Integer in its range, or a
+      #   +:string+ gets a value that is not valid UTF-8 and cannot be converted to it
       def write_value(type, value)
         case type
-        when :byte then @buf << (value & 0xFF)
-        when :i16, :i32, :i64 then write_zigzag(value)
+        when :byte then @buf << (checked_int(type, value) & 0xFF)
+        when :i16, :i32, :i64 then write_zigzag(checked_int(type, value))
         when :double then @buf << [value].pack("E")
-        when :binary, :string then write_binary(value)
+        when :binary then write_binary(value)
+        when :string then write_binary(utf8(value))
         when Array then write_list(type[1], value)
         else write_struct(value)
         end
@@ -440,6 +447,32 @@ module Herringbone
           end
         end
       end
+
+      private
+
+      # @param type [Symbol] declared integer type, a key of INT_RANGES
+      # @param value [Object] value to be written as +type+
+      # @return [Integer] +value+
+      # @raise [Error] when +value+ is not an Integer that +type+ can hold
+      def checked_int(type, value)
+        raise Error, "#{value.inspect} is not a valid #{type}" unless value.is_a?(Integer) && INT_RANGES.fetch(type).cover?(value)
+        value
+      end
+
+      # Parquet strings are UTF-8. Binary Strings are taken to hold UTF-8 bytes, Strings in other
+      # encodings are converted.
+      #
+      # @param s [String] value of a +:string+ field
+      # @return [String] +s+, or a UTF-8 copy of it
+      # @raise [Error] when +s+ is not valid UTF-8 and cannot be converted to it
+      def utf8(s)
+        return s if s.ascii_only? || (s.encoding == Encoding::UTF_8 && s.valid_encoding?)
+        u = (s.encoding == Encoding::BINARY) ? s.dup.force_encoding(Encoding::UTF_8) : s.encode(Encoding::UTF_8)
+        raise Error, "String in #{s.encoding} is not valid UTF-8" unless u.valid_encoding?
+        u
+      rescue EncodingError => e
+        raise Error, "String in #{s.encoding} cannot be converted to UTF-8: #{e.message}"
+      end
     end
 
     # A field declared on a Thrift::Struct subclass
@@ -458,14 +491,14 @@ module Herringbone
     #   field 1, :name, :i32
     class Struct
       class << self
-        # @return [Array<Field>] declared fields, sorted by id
+        # @return [Array<Field>] declared fields, sorted by id, including the superclass's
         def fields
-          @fields || []
+          @fields || ((superclass < Struct) ? superclass.fields : [])
         end
 
         # @return [Hash{Integer => Field}] declared fields by id, for decoding
         def fields_by_id
-          @fields_by_id || {}
+          @fields_by_id || ((superclass < Struct) ? superclass.fields_by_id : {})
         end
 
         # Declares a field and defines its accessor
@@ -534,6 +567,12 @@ module Herringbone
       def ==(other)
         other.class == self.class && other.to_h == to_h
       end
+      alias_method :eql?, :==
+
+      # @return [Integer] hash agreeing with #==, so equal structs work as Hash keys and in +uniq+
+      def hash
+        [self.class, to_h].hash
+      end
 
       # @return [String] short class name and the set fields
       def inspect

data/lib/herringbone/types.rb

@@ -18,6 +18,8 @@ module Herringbone
   #   DECIMAL               <-> BigDecimal
   #   UUID                  <-> String "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
   #   FLOAT16               <-> Float
+  #
+  # @api private
   module Types
     module_function
 
@@ -81,13 +83,15 @@ module Herringbone
     end
 
     # Physical attributes (type, logical type etc.) for a DSL type name
+    #
+    # +:time+ and +:timestamp+ hold values converted to UTC and are flagged isAdjustedToUTC;
+    # +:local_time+ and +:local_timestamp+ hold wall clock values, are not flagged and get no
+    # legacy converted type.
     # @param type [Symbol, String] column type name, e.g. +:string+, +:int64+, +:decimal+, +:timestamp+
     # @param opts [Hash{Symbol => Object}] type-specific options
     # @option opts [Boolean] :parquet_enum for +:enum+, annotate as ENUM instead of STRING
     # @option opts [Integer] :length byte length for +:fixed+ (required)
-    # @option opts [Symbol] :unit (:micros) for +:time+ and +:timestamp+, see {time_unit}
-    # @option opts [Boolean] :utc (true) for +:time+ and +:timestamp+, sets +is_adjusted_to_utc+;
-    #   the legacy converted type is only written when true
+    # @option opts [Symbol] :unit (:micros) for the time and timestamp types, see {time_unit}
     # @option opts [Integer] :precision number of decimal digits for +:decimal+ (required)
     # @option opts [Integer] :scale (0) digits after the decimal point for +:decimal+
     # @option opts [Symbol] :physical for +:decimal+, force +:int32+, +:int64+, +:binary+ or
@@ -125,16 +129,18 @@ module Herringbone
       when :int96 then {type: T::INT96}
       when :fixed
         {type: T::FIXED_LEN_BYTE_ARRAY, type_length: Integer(opts.fetch(:length))}
-      when :time
+      when :time, :local_time
+        utc = type.to_sym == :time
         unit = time_unit(opts.fetch(:unit, :micros))
         converted = {millis: C::TIME_MILLIS, micros: C::TIME_MICROS}[unit.to_sym]
-        {type: unit.millis ? T::INT32 : T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
-         logical_type: lt(time: Format::TimeType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit))}
-      when :timestamp
+        {type: unit.millis ? T::INT32 : T::INT64, converted_type: utc ? converted : nil,
+         logical_type: lt(time: Format::TimeType.new(is_adjusted_to_utc: utc, unit: unit))}
+      when :timestamp, :local_timestamp
+        utc = type.to_sym == :timestamp
         unit = time_unit(opts.fetch(:unit, :micros))
         converted = {millis: C::TIMESTAMP_MILLIS, micros: C::TIMESTAMP_MICROS}[unit.to_sym]
-        {type: T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
-         logical_type: lt(timestamp: Format::TimestampType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit))}
+        {type: T::INT64, converted_type: utc ? converted : nil,
+         logical_type: lt(timestamp: Format::TimestampType.new(is_adjusted_to_utc: utc, unit: unit))}
       when :decimal
         precision = Integer(opts.fetch(:precision))
         scale = Integer(opts.fetch(:scale, 0))
@@ -335,7 +341,8 @@ module Herringbone
     #   date:      Date, Time/DateTime (its calendar date), ISO-8601 String, Integer days since epoch
     #   timestamp: Time, DateTime, ActiveSupport::TimeWithZone, Date (midnight UTC), ISO-8601 String,
     #              Integer in the column's unit
-    #   time:      Time (its time of day), "HH:MM[:SS[.fraction]]" String, Integer in the column's unit
+    #   time:      Time (its time of day, in UTC unless the column is local), "HH:MM[:SS[.fraction]]"
+    #              String, Integer in the column's unit
     #   json:      String (used as-is) or any other object (serialized with JSON.generate)
     #   string:    String, Symbol or anything responding to to_s
     #   boolean:   true/false, 1/0, "true"/"false", "t"/"f", "1"/"0", "yes"/"no"
@@ -355,7 +362,7 @@ module Herringbone
       when :timestamp
         return timestamp_writer(a, b)
       when :time
-        return time_writer(a) if type == T::INT32 || type == T::INT64
+        return time_writer(a, b) if type == T::INT32 || type == T::INT64
       when :json
         return ->(v) { v.is_a?(String) ? v : JSON.generate(v) }
       when :string, :enum
@@ -481,9 +488,11 @@ module Herringbone
     # anything responding to +hour+ and +nsec+ (Time, DateTime), and raises ArgumentError or
     # RangeError for anything else or a time past 23:59:59.
     # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
+    # @param utc [Boolean] true to convert a Time to UTC before taking its time of day, false to
+    #   take the time of day in the Time's own zone
     # @return [Proc] one-argument lambda
     # @raise [KeyError] for an unknown unit
-    def time_writer(unit)
+    def time_writer(unit, utc)
       mult = UNIT_DIVISORS.fetch(unit)
       lambda do |v|
         return v if v.is_a?(Integer)
@@ -494,6 +503,7 @@ module Herringbone
           secs = m[1].to_i * 3600 + m[2].to_i * 60 + m[3].to_i
         else
           raise ArgumentError, "expected a Time, got #{v.class}" unless v.respond_to?(:hour) && v.respond_to?(:nsec)
+          v = v.getutc if utc && v.respond_to?(:getutc)
           nanos = v.nsec
           secs = v.hour * 3600 + v.min * 60 + v.sec
         end

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.6.1"
+  VERSION = "0.8.0"
 end

data/lib/herringbone/visualizer.rb

@@ -15,6 +15,8 @@ module Herringbone
   # Used through Inspector#to_html:
   #
   #   File.open("data.parquet", "rb") { |io| Herringbone::Inspector.new(io).to_html }
+  #
+  # @api private
   class Visualizer
     # highlight.js build loaded by the page to colour the footer JSON; optional.
     HIGHLIGHT_JS = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"

data/lib/herringbone/writer.rb

@@ -76,6 +76,8 @@ module Herringbone
     MAX_DICTIONARY_BYTES = 1024 * 1024
     # The row group byte size is first estimated after this many rows, then after every row group
     ESTIMATE_AFTER_ROWS = 1000
+    # Options .open_for_copies refuses, since the row groups it writes keep their boundaries
+    ROW_GROUP_OPTIONS = %i[row_group_bytes row_group_rows].freeze
 
     # @return [Schema] schema the rows are written with
     attr_reader :schema
@@ -121,6 +123,29 @@ module Herringbone
       result
     end
 
+    # Internal (used by Redaction and Combiner): .open for a file whose row groups are all written
+    # with #write_row_group, one for each row group of another file, so they keep their boundaries
+    #
+    # @param io [IO, #write] destination
+    # @param schema [Schema] schema of the rows
+    # @param refusal [String] why +row_group_bytes:+ and +row_group_rows:+ cannot be given, e.g.
+    #   "a redaction keeps the row groups of the input"
+    # @param options [Hash{Symbol => Object}] see .open, without the row group options
+    # @option options [Symbol] :compression (:snappy) codec of the chunks encoded again
+    # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
+    # @option options [EncryptionConfiguration, Hash{Symbol => Object}, nil] :encryption (nil) see #initialize
+    # @yield [writer] the open writer
+    # @yieldparam writer [Writer] writer to write row groups to
+    # @yieldreturn [Object] returned by open_for_copies
+    # @return [Object] the block's value
+    # @raise [ArgumentError] for +row_group_bytes:+ / +row_group_rows:+, or an invalid option
+    # @api private
+    def self.open_for_copies(io, schema, refusal, **options, &block)
+      bad = options.keys & ROW_GROUP_OPTIONS
+      raise ArgumentError, "#{bad.join(", ")}: #{refusal}" unless bad.empty?
+      self.open(io, schema, row_group_bytes: 1 << 62, **options, &block)
+    end
+
     # Validates the options and writes the leading magic bytes to +io+.
     #
     # @param io [IO, #write] destination; switched to binmode when it supports that
@@ -269,11 +294,12 @@ module Herringbone
     #   @return [Format::OffsetIndex, nil] the chunk's OffsetIndex, with source file offsets
     # @!attribute bloom_filter
     #   @return [String, nil] the chunk's encoded bloom filter, header included
+    # @api private
     CopiedChunk = Struct.new(:chunk, :start, :bytes, :column_index, :offset_index, :bloom_filter)
 
-    # Internal (used by Redaction): writes a row group of +num_rows+ rows from the buffered values,
-    # except for the columns in +copies+, whose chunks are copied byte for byte from another file
-    # with their offsets rebased. Starts a new row group afterwards.
+    # Internal (used by Redaction and Combiner): writes a row group of +num_rows+ rows from the
+    # buffered values, except for the columns in +copies+, whose chunks are copied byte for byte
+    # from another file with their offsets rebased. Starts a new row group afterwards.
     #
     # @param num_rows [Integer] rows in the row group
     # @param copies [Hash{Integer => CopiedChunk}] column index => chunk to copy instead of encoding
@@ -285,6 +311,7 @@ module Herringbone
     # @param sorting_columns [Array<Format::SortingColumn>, nil] stored on the RowGroup as given
     # @return [void]
     # @raise [Error] when the writer is closed
+    # @api private
     def write_row_group(num_rows, copies: {}, codecs: {}, bloom_filters: {}, sorting_columns: nil)
       raise Error, "Writer is closed" if @closed
       if @encryption && (encrypted = copies.keys.find { |i| @encryption.encrypted?(i) })
@@ -293,7 +320,7 @@ module Herringbone
       start = @pos
       chunks = @schema.columns.map do |col|
         if (copy = copies[col.index])
-          copy_column_chunk(copy)
+          copy_column_chunk(col, copy)
         else
           bloom = @bloom_filters[col.dotted_path]
           bloom ||= bloom_filter_settings({}) if bloom_filters[col.index] && BloomFilter::TYPES.include?(col.type)
@@ -314,21 +341,23 @@ module Herringbone
       reset_buffers
     end
 
-    # Internal (used by Redaction): whether the column is encrypted in this file
+    # Internal (used by Redaction and Combiner): whether the column is encrypted in this file
     #
     # @param index [Integer] leaf column index
     # @return [Boolean]
+    # @api private
     def encrypted_column?(index) = @encryption&.encrypted?(index) || false
 
-    # Internal (used by Redaction): shreds one value per row of a top-level field into the buffers,
-    # for #write_row_group. Other fields are left as they are, so the caller decides which columns
-    # are encoded and which are copied.
+    # Internal (used by Redaction and Combiner): shreds one value per row of a top-level field into
+    # the buffers, for #write_row_group. Other fields are left as they are, so the caller decides
+    # which columns are encoded and which are copied.
     #
     # @param name [String] top-level field name
     # @param values [Array] the field's value for each row
     # @return [void]
     # @raise [ArgumentError] when the schema has no such field
     # @raise [EncodeError] when a value cannot be written to the field
+    # @api private
     def buffer_field(name, values)
       field = @schema.field(name) or raise ArgumentError, "No such field #{name.inspect}"
       values.each_with_index do |value, i|
@@ -414,6 +443,7 @@ module Herringbone
     #   @return [String, Array<Integer>, nil] repetition levels, like +defs+; nil for non-repeated columns
     # @!attribute values
     #   @return [Array, ByteValues] non-null physical values
+    # @api private
     ColumnBuffer = Struct.new(:defs, :reps, :values)
 
     # Starts a new row group: fresh buffers per column, and the per-field write plan
@@ -769,6 +799,7 @@ module Herringbone
     #   @return [Integer] non-null values in the page
     # @!attribute range
     #   @return [Array(Object, Object), nil] [min, max] physical values, nil when there are none
+    # @api private
     PageInfo = Struct.new(:offset, :size, :first_row, :nulls, :non_null, :range)
 
     # @param pages [Array<PageInfo>] pages of a column chunk
@@ -781,14 +812,17 @@ module Herringbone
 
     # Writes a chunk copied from another file. The pages, the ColumnIndex and the bloom filter
     # hold no file offsets and go out as they are; the ColumnMetaData and the OffsetIndex do, so
-    # those are rebased onto where the chunk lands in this file.
+    # those are rebased onto where the chunk lands in this file. The path is this file's, which
+    # differs when the source file names list elements another way ("list.item", "array").
     #
+    # @param col [Schema::Column] the column the chunk is copied into
     # @param copy [CopiedChunk] the chunk to copy
     # @return [Format::ColumnChunk] chunk with its ColumnMetaData, for the row group
-    def copy_column_chunk(copy)
+    def copy_column_chunk(col, copy)
       start = @pos
       shift = start - copy.start
       meta = Format::ColumnMetaData.decode(copy.chunk.meta_data.encode).first
+      meta.path_in_schema = col.path
       meta.data_page_offset += shift
       dict = meta.dictionary_page_offset
       # Some writers store 0 when there is no dictionary page
@@ -1070,7 +1104,7 @@ module Herringbone
     # keeps in their source codec use that codec's default
     #
     # @param codec [Integer] codec id
-    # @param data [String] bytes to compress
+    # @param data [String, Array<String>] bytes to compress, or the parts of them
     # @return [String] compressed bytes
     def compress(codec, data)
       Compression.compress(codec, data, (codec == @codec) ? @compression_level : nil)
@@ -1080,28 +1114,30 @@ module Herringbone
     # stored, encrypted.
     #
     # @param header [Format::PageHeader] header; sizes and CRC32 are filled in here
-    # @param body [String] uncompressed page body
-    # @param compressed [String, nil] bytes to write as the page body, when already prepared (v2 data
-    #   pages, whose levels stay uncompressed); +body+ is compressed otherwise
+    # @param body [String, Array<String>] uncompressed page body, or its parts
+    # @param compressed [String, Array<String>, nil] bytes to write as the page body, or their
+    #   parts, when already prepared (v2 data pages, whose levels stay uncompressed); +body+ is
+    #   compressed otherwise
     # @param codec [Integer] codec id to compress +body+ with
     # @param crypto [Encryption::ModuleCrypto, nil] encryption of the column's modules
     # @param ordinal [Integer, nil] data page ordinal within the chunk; nil for a dictionary page
     # @return [Integer]
     def write_page(header, body, compressed = nil, codec: @codec, crypto: nil, ordinal: nil)
-      compressed ||= compress(codec, body)
-      header.uncompressed_page_size ||= body.bytesize
+      parts = Array(compressed || compress(codec, body))
+      header.uncompressed_page_size ||= Array(body).sum(&:bytesize)
       dictionary = header.type == Format::PageType::DICTIONARY_PAGE
       if crypto
-        compressed = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE, compressed, ordinal)
+        parts = [crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE, parts.join, ordinal)]
       end
-      header.compressed_page_size = compressed.bytesize
-      header.crc = Zlib.crc32(compressed).then { |c| (c >= 0x8000_0000) ? c - 0x1_0000_0000 : c }
+      header.compressed_page_size = parts.sum(&:bytesize)
+      crc = parts.inject(0) { |c, part| Zlib.crc32(part, c) }
+      header.crc = (crc >= 0x8000_0000) ? crc - 0x1_0000_0000 : crc
       encoded = header.encode
       if crypto
         encoded = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE_HEADER : Encryption::DATA_PAGE_HEADER, encoded, ordinal)
       end
       write_raw(encoded)
-      write_raw(compressed)
+      parts.each { |part| write_raw(part) unless part.empty? }
       encoded.bytesize + header.uncompressed_page_size
     end
 
@@ -1117,7 +1153,7 @@ module Herringbone
     #   modules and the page's ordinal, for an encrypted column
     # @return [Integer] uncompressed size including the header
     def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding, codec, page_crypto)
-      body = String.new(encoding: Encoding::BINARY)
+      body = []
       body << [rep_bytes.bytesize].pack("V") << rep_bytes unless rep_bytes.empty?
       body << [def_bytes.bytesize].pack("V") << def_bytes unless def_bytes.empty?
       body << encoded
@@ -1156,7 +1192,7 @@ module Herringbone
           is_compressed: codec != Format::Codec::UNCOMPRESSED
         )
       )
-      write_page(header, "".b, rep_bytes + def_bytes + compressed, codec: codec, crypto: page_crypto&.first,
+      write_page(header, "".b, [rep_bytes, def_bytes, compressed], codec: codec, crypto: page_crypto&.first,
         ordinal: page_crypto&.last)
     end
 

data/lib/herringbone/xxhash.rb

@@ -14,6 +14,8 @@ module Herringbone
   # allocations trigger so many garbage collections that hashing becomes up to 10x slower. The
   # arithmetic is written once with the small code generators below, whose output is inlined into
   # the hashing methods: no method calls or allocations in the hot paths.
+  #
+  # @api private
   module XXHash
     # 64-bit mask
     M = 0xFFFF_FFFF_FFFF_FFFF

data/lib/herringbone.rb

@@ -40,6 +40,34 @@ module Herringbone
   # such as a codec whose library is unavailable
   class UnsupportedError < Error; end
 
+  # Schemas that do not fit together: two schemas that cannot be united or intersected
+  # (Schema#union, Schema#intersect), or inputs of Herringbone.combine that do not fit the output
+  # schema or each other. The message lists every field that does not fit, one per line, e.g.
+  #
+  #   Cannot unite the schemas, 2 fields do not fit:
+  #     price: int64 vs double (int64 does not fit a double exactly)
+  #     address.zip: int32 vs string (no common type)
+  class IncompatibleSchema < Error
+    # A field that does not fit: its dotted path, how each side declares it, why the two do not
+    # fit, and for Herringbone.combine the position of the input it was found in
+    Conflict = Struct.new(:path, :left, :right, :reason, :input) do
+      # @return [String] e.g. "address.zip: int32 vs string (no common type)"
+      def to_s = "#{path.empty? ? "(top level)" : path}: #{left} vs #{right} (#{reason})"
+    end
+
+    # @return [Array<Conflict>] every field that does not fit, in schema (and input) order
+    attr_reader :conflicts
+
+    # @param conflicts [Array<Conflict>] the fields that do not fit
+    # @param operation [String, nil] what was attempted, "unite" or "intersect", for the message
+    # @param message [String, nil] the message, instead of one built from +operation+
+    def initialize(conflicts, operation: nil, message: nil)
+      @conflicts = conflicts
+      count = (conflicts.size == 1) ? "1 field does" : "#{conflicts.size} fields do"
+      super(message || "Cannot #{operation} the schemas, #{count} not fit:\n#{conflicts.map { |c| "  #{c}" }.join("\n")}")
+    end
+  end
+
   # An encrypted file or column cannot be read: its key was not given, the key or AAD prefix is
   # wrong, or the encrypted bytes were changed
   class DecryptionError < Error; end
@@ -56,6 +84,7 @@ module Herringbone
 
   autoload :BloomFilter, "#{LIB}/bloom_filter"
   autoload :ByteValues, "#{LIB}/byte_values"
+  autoload :Combiner, "#{LIB}/combiner"
   autoload :Compression, "#{LIB}/compression"
   autoload :DecryptionConfiguration, "#{LIB}/encryption_configuration"
   autoload :Encryption, "#{LIB}/encryption"
@@ -68,6 +97,7 @@ module Herringbone
   autoload :MissingCodecError, "#{LIB}/compression"
   autoload :Reader, "#{LIB}/reader"
   autoload :Redaction, "#{LIB}/redaction"
+  autoload :RestrictedReadableIO, "#{LIB}/restricted_readable_io"
   autoload :Schema, "#{LIB}/schema"
   autoload :SimpleWriter, "#{LIB}/simple_writer"
   autoload :Thrift, "#{LIB}/thrift"
@@ -176,16 +206,17 @@ module Herringbone
     writer.rows_written
   end
 
-  # Rewrites +input+ into +output+ with rows removed or values replaced, building the Redaction from
-  # the block (or taking +redaction+). A shortcut for Redaction#apply.
+  # Rewrites +io_or_reader+ into +output_io+ with rows removed or values replaced, building the
+  # Redaction from the block (or taking +redaction+). A shortcut for Redaction#apply.
   #
-  #   Herringbone.redact(input, output) do |r|
+  #   Herringbone.redact(io, output_io) do |r|
   #     r.where(user_id: 42).delete
   #     r.replace(:email) { |email| email && OpenSSL::HMAC.hexdigest("SHA256", KEY, email) }
   #   end
   #
-  # @param input [IO, StringIO] the Parquet file, read with #seek and #read; not closed
-  # @param output [IO, #write] destination, written sequentially; not closed
+  # @param io_or_reader [IO, StringIO, Reader] the Parquet file, read with #seek and #read, or a
+  #   Reader of it, whose IO and decryption are used; not closed
+  # @param output_io [IO, #write] destination, written sequentially; not closed
   # @param redaction [Redaction, nil] the redaction to apply, instead of a block
   # @param writer_options [Hash{Symbol => Object}] Writer options for re-encoded column chunks, and
   #   +metadata:+ to replace the footer key/value metadata, see Redaction#apply
@@ -204,15 +235,71 @@ module Herringbone
   # @yieldreturn [void]
   # @return [Redaction::Report] what was done
   # @raise [ArgumentError] when given both or neither of +redaction+ and a block, a block that takes
-  #   no parameter, or when the redaction does not fit the file's schema
+  #   no parameter, a Reader with +decryption:+, or when the redaction does not fit the file's schema
   # @raise [EncodeError] when a replacement value cannot be written to its column
-  def redact(input, output, redaction = nil, **writer_options, &block)
+  def redact(io_or_reader, output_io, redaction = nil, **writer_options, &block)
     raise ArgumentError, "Herringbone.redact takes a Redaction or a block, not both" if redaction && block
     raise ArgumentError, "Herringbone.redact needs a Redaction or a block" unless redaction || block
     if block&.arity&.zero?
-      raise ArgumentError, "The block receives the redaction: Herringbone.redact(input, output) { |r| r.where(user_id: 42).delete }"
+      raise ArgumentError, "The block receives the redaction: Herringbone.redact(io, output_io) { |r| r.where(user_id: 42).delete }"
     end
-    (redaction || Redaction.new(&block)).apply(input, output, **writer_options)
+    (redaction || Redaction.new(&block)).apply(io_or_reader, output_io, **writer_options)
+  end
+
+  # Concatenates Parquet files into +output_io+: every row group of every input, in order. A
+  # shortcut for Combiner. Column chunks are copied byte for byte (only offsets are rebased)
+  # wherever the output stores a column's values as the input does, so files with the same schema
+  # are combined without decoding anything.
+  #
+  #   Herringbone.combine([jan, feb, mar], output_io)                      # the same schema
+  #   Herringbone.combine([old, new], output_io, schema: :union)           # all fields, nulls in the gaps
+  #   Herringbone.combine([a, b], output_io, schema: :intersect)           # the fields all inputs have
+  #
+  # Without +schema:+ every input must have the schema of the first (see Schema#==). With
+  # +schema: :union+ the output has the fields of all inputs, widened as Schema#union widens them,
+  # and fields an input lacks are written as nulls. With +schema: :intersect+ it has the fields all
+  # inputs have, and the others are dropped. A Schema given as +schema:+ must hold every input: each
+  # must have the same fields or fewer, in the same or narrower types. Every input is checked
+  # before anything is written, and IncompatibleSchema lists all that do not fit.
+  #
+  # The output is encrypted like the encrypted inputs, if any (plaintext inputs included), unless
+  # +encryption:+ says otherwise; +encryption: false+ writes it in plaintext. Inputs encrypted
+  # differently need +encryption:+. Herringbone never opens files by path: pass open IOs, which
+  # are read twice (footers first, then chunks), so keep them open until +combine+ returns and
+  # mind the limit on open files when combining very many of them.
+  #
+  # @param ios_or_readers [Enumerable<IO, StringIO, Reader>] the Parquet files: IOs read with #seek
+  #   and #read, or Readers, which bring their own decryption; enumerated once, none is closed
+  # @param output_io [IO, #write] destination, written sequentially; not closed
+  # @param schema [Schema, Symbol, nil] +:union+, +:intersect+, a Schema every input fits, or nil
+  #   to require the inputs to have the same schema
+  # @param decryption [DecryptionConfiguration, Hash{Symbol => Object}, Array, #call, nil] keys of
+  #   encrypted inputs given as IOs, see Reader.new
+  # @param writer_options [Hash{Symbol => Object}] Writer options for chunks encoded again (and
+  #   written nulls), +metadata:+ and +encryption:+
+  # @option writer_options [Symbol] :compression (codec of each source chunk) codec for chunks
+  #   encoded again
+  # @option writer_options [Integer, nil] :compression_level (nil) level for that codec, see Writer
+  # @option writer_options [Hash{String => String}] :metadata (the first input's) footer key/value metadata
+  # @option writer_options [EncryptionConfiguration, Hash{Symbol => Object}, Key, String, false] :encryption
+  #   (encrypted like the encrypted inputs) see Writer; false for a plaintext output
+  # @option writer_options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
+  #   columns whose encoded chunks get a bloom filter, besides those whose source chunk had one
+  # @option writer_options [Integer] :page_bytes (1MB) approximate uncompressed data page size
+  # @option writer_options [Integer] :page_rows (20_000) maximum rows per data page
+  # @option writer_options [Integer] :data_page_version (1) 1 or 2
+  # @option writer_options [Boolean, Array<String>] :dictionary (true) see Writer
+  # @option writer_options [Hash{String => Symbol}] :encodings ({}) see Writer
+  # @return [Combiner::Report] rows written, how many row groups were copied or rewritten, and per
+  #   input which fields were filled with nulls, widened or dropped
+  # @raise [IncompatibleSchema] listing every input (and field) that does not fit the others or
+  #   +schema:+
+  # @raise [ArgumentError] when there are no inputs, an input is neither an IO nor a Reader, the
+  #   encrypted inputs are encrypted differently and +encryption:+ is not given, or for
+  #   +row_group_bytes:+ / +row_group_rows:+
+  # @raise [DecryptionError] when an input is encrypted and a key it needs was not given
+  def combine(ios_or_readers, output_io, schema: nil, decryption: nil, **writer_options)
+    Combiner.new(ios_or_readers, schema: schema, decryption: decryption).apply(output_io, **writer_options)
   end
 
   # Compression codecs this process can read and write, e.g. [:none, :snappy, :gzip, :lz4, :lz4_hadoop, :zstd].

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: herringbone
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 bindir: bin
 cert_chain: []
-date: 2026-10-04 00:00:00.000000000 Z
+date: 2026-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -45,6 +45,7 @@ files:
 - lib/herringbone/codecs/lz4.rb
 - lib/herringbone/codecs/lzo.rb
 - lib/herringbone/codecs/snappy.rb
+- lib/herringbone/combiner.rb
 - lib/herringbone/compression.rb
 - lib/herringbone/encodings/delta.rb
 - lib/herringbone/encodings/plain.rb
@@ -58,6 +59,7 @@ files:
 - lib/herringbone/key.rb
 - lib/herringbone/reader.rb
 - lib/herringbone/reader/bloom_filters.rb
+- lib/herringbone/reader/chunk_copier.rb
 - lib/herringbone/reader/column_chunk_reader.rb
 - lib/herringbone/reader/column_cursor.rb
 - lib/herringbone/reader/numo.rb
@@ -65,7 +67,9 @@ files:
 - lib/herringbone/reader/scan.rb
 - lib/herringbone/redaction.rb
 - lib/herringbone/redaction/rewriter.rb
+- lib/herringbone/restricted_readable_io.rb
 - lib/herringbone/schema.rb
+- lib/herringbone/schema/merge.rb
 - lib/herringbone/simple_writer.rb
 - lib/herringbone/thrift.rb
 - lib/herringbone/types.rb