herringbone 0.6.0 → 0.7.0

data/lib/herringbone/inspector.rb

@@ -29,6 +29,8 @@ module Herringbone
     WINDOW = 64 * 1024
 
     # Just enough of parquet.thrift's BloomFilterHeader to learn the filter's size
+    #
+    # @api private
     class BloomFilterHeader < Thrift::Struct
       field 1, :num_bytes, :i32
     end
@@ -484,13 +486,13 @@ module Herringbone
     # @raise [FormatError] when the file is too small, lacks the magic bytes or has a corrupt footer
     # @raise [DecryptionError] when the footer is encrypted and cannot be decrypted
     def initialize(io, decryption: nil)
-      @io = io
       @decryption = decryption.nil? ? nil : DecryptionConfiguration.from(decryption)
-      unless @io.respond_to?(:read) && @io.respond_to?(:seek)
+      unless io.respond_to?(:read) && io.respond_to?(:seek)
         raise ArgumentError, "Herringbone::Inspector expects an IO that supports #seek and #read " \
           "(e.g. File.open(path, \"rb\")), got #{io.class}"
       end
-      @name = (@io.respond_to?(:path) && @io.path) ? File.basename(@io.path.to_s) : nil
+      @io = RestrictedReadableIO.wrap(io)
+      @name = File.basename(@io.path) unless @io.path.equal?(RestrictedReadableIO::UNTITLED)
       read_footer
       @schema = Schema.from_elements(@metadata.schema)
     end
@@ -907,6 +909,7 @@ module Herringbone
     # Decryption of an encrypted chunk's modules
     # @param chunk [ColumnChunkInfo] an encrypted chunk
     # @return [Encryption::ModuleCrypto, nil] nil when the chunk's key was not given
+    # @api private
     def chunk_crypto(chunk)
       @decryptor&.chunk(chunk.row_group.index, chunk.row_group.row_group.ordinal, chunk.column, chunk.chunk)
     rescue DecryptionError
@@ -918,6 +921,7 @@ module Herringbone
     # @param chunk [ColumnChunkInfo] chunk whose pages to walk
     # @return [Array(Array<PageInfo>, String), Array(Array<PageInfo>, nil)] the pages found, and why
     #   the walk stopped early (nil when every value was accounted for)
+    # @api private
     def walk_pages(chunk)
       return [[], "column chunk stored in external file #{chunk.external_file}"] if chunk.external_file
       if chunk.encrypted?
@@ -959,6 +963,7 @@ module Herringbone
     # Reads and decodes a chunk's ColumnIndex, decoding the per-page min/max with the column type
     # @param chunk [ColumnChunkInfo] chunk whose ColumnIndex to read
     # @return [ColumnIndexInfo, nil] nil when the chunk has none or it fails to decode
+    # @api private
     def read_column_index(chunk)
       offset, length = chunk.column_index_range
       return nil unless offset && length.positive?
@@ -985,6 +990,7 @@ module Herringbone
     # Reads and decodes a chunk's OffsetIndex
     # @param chunk [ColumnChunkInfo] chunk whose OffsetIndex to read
     # @return [OffsetIndexInfo, nil] nil when the chunk has none or it fails to decode
+    # @api private
     def read_offset_index(chunk)
       offset, length = chunk.offset_index_range
       return nil unless offset && length.positive?
@@ -1006,6 +1012,7 @@ module Herringbone
     # @param offset [Integer] file offset of the BloomFilterHeader
     # @param encrypted [Boolean] whether the filter is encrypted
     # @return [Integer, nil] nil when the header can't be decoded or has no num_bytes
+    # @api private
     def bloom_filter_size(offset, encrypted = false)
       if encrypted
         header = read_at(offset, 4).unpack1("V") or return nil
@@ -1022,6 +1029,7 @@ module Herringbone
     # :ok, :mismatch or :absent for one page (reads its body). Sets PageInfo#actual_crc.
     # @param page [PageInfo] page to check
     # @return [Symbol]
+    # @api private
     def page_checksum(page)
       return :absent unless page.crc
       page.actual_crc = page_crc(page)
@@ -1031,6 +1039,7 @@ module Herringbone
     # CRC32 of a page's body as stored
     # @param page [PageInfo] page whose compressed body to read
     # @return [Integer] unsigned 32-bit CRC
+    # @api private
     def page_crc(page)
       Zlib.crc32(read_at(page.body_offset, page.compressed_size))
     end
@@ -1039,6 +1048,7 @@ module Herringbone
     # @param chunk [ColumnChunkInfo] chunk whose page headers to compare with its ColumnIndex
     # @return [Array<Hash{Symbol => Object}>] one entry per disagreement; a single :page_count
     #   entry when the ColumnIndex and the data pages differ in number
+    # @api private
     def compare_page_index(chunk)
       ci = chunk.column_index or return []
       data = chunk.data_pages
@@ -1072,6 +1082,7 @@ module Herringbone
     # @param st [Format::Statistics, nil] statistics from column metadata or a page header
     # @param column [Schema::Column] column the statistics describe
     # @return [Stats, nil] nil when +st+ is nil
+    # @api private
     def decode_statistics(st, column)
       return nil unless st
       order = Inspector.sort_order(column)
@@ -1111,6 +1122,7 @@ module Herringbone
     # @param column [Schema::Column] column whose physical type and converter apply
     # @return [Object, nil] the converted value, a hex String when it could not be decoded, nil
     #   for nil (or empty BOOLEAN) input
+    # @api private
     def decode_value(bytes, column)
       return nil if bytes.nil?
       bytes = bytes.b
@@ -1158,6 +1170,8 @@ module Herringbone
 
       # A minimal flatbuffer reader: tables (through their vtables), scalars, strings, vectors of
       # scalars and tables, and unions. Every read is bounds-checked; malformed input raises Error.
+      #
+      # @api private
       class FlatBuffer
         # @param bytes [String] the flatbuffer (copied as binary)
         def initialize(bytes)
@@ -1241,6 +1255,8 @@ module Herringbone
       end
 
       # One flatbuffer table; fields are addressed by their slot (declaration order in the .fbs)
+      #
+      # @api private
       class Table
         # @param fb [FlatBuffer] buffer the table lives in
         # @param pos [Integer] absolute position of the table (where its vtable offset is stored)
@@ -1491,11 +1507,13 @@ module Herringbone
 
     # @param e [Integer] Parquet Encoding value
     # @return [String] its name, e.g. "RLE_DICTIONARY" (the number as a String when unknown)
+    # @api private
     def self.encoding_name(e) = Format::Encoding::NAMES[e]&.to_s || e.to_s
 
     # @param column [Schema::Column] leaf column
     # @return [String] physical type with its logical annotation, e.g. "BYTE_ARRAY STRING" or
     #   "FIXED_LEN_BYTE_ARRAY(16) UUID"
+    # @api private
     def self.type_name(column)
       node = column.node
       phys = T::NAMES[node.type].to_s
@@ -1508,6 +1526,7 @@ module Herringbone
     # @param node [Schema::Node] schema node (leaf or group)
     # @return [String, nil] e.g. "INTEGER(8, unsigned)", "TIMESTAMP(MICROS, UTC)" or "DECIMAL(10, 2)";
     #   nil when the node has no annotation
+    # @api private
     def self.logical_type_name(node)
       if (kind = node.logical_type&.kind)
         name, payload = kind
@@ -1528,6 +1547,7 @@ module Herringbone
     # :signed, :unsigned or :unknown, per the Parquet sort order rules for the column's type
     # @param column [Schema::Column] leaf column
     # @return [Symbol]
+    # @api private
     def self.sort_order(column)
       kind, _a, signed = Types.logical_of(column.node)
       case kind
@@ -1547,6 +1567,7 @@ module Herringbone
     # Recurses into Hashes, Arrays and Structs; Hash keys other than Symbols become Strings.
     # @param v [Object] value to convert
     # @return [Hash, Array, String, Integer, Float, Boolean, nil]
+    # @api private
     def self.jsonable(v)
       case v
       when Hash then v.each_with_object({}) { |(k, x), h| h[k.is_a?(Symbol) ? k : k.to_s] = jsonable(x) }
@@ -1567,6 +1588,7 @@ module Herringbone
     # Strings already tagged as valid UTF-8 are returned as they are, control characters and all.
     # @param s [String] string in any encoding
     # @return [String]
+    # @api private
     def self.text(s)
       return s if s.encoding == Encoding::UTF_8 && s.valid_encoding?
       u = s.dup.force_encoding(Encoding::UTF_8)
@@ -1577,6 +1599,7 @@ module Herringbone
     # @param bytes [String] bytes to show
     # @return [String] "0x" and lowercase hex digits; only the first 64 bytes, followed by the
     #   total size, for longer input
+    # @api private
     def self.hex(bytes)
       b = bytes.b
       (b.bytesize > 64) ? "0x#{b.byteslice(0, 64).unpack1("H*")}… (#{b.bytesize} bytes)" : "0x#{b.unpack1("H*")}"
@@ -1587,6 +1610,7 @@ module Herringbone
     # @param v [Object] decoded value
     # @param max [Integer] longest result, in characters; longer ones are cut and end in an ellipsis
     # @return [String]
+    # @api private
     def self.display(v, max: 40)
       s = case v
       when String then (v.encoding == Encoding::BINARY) ? text(v) : v
@@ -1599,6 +1623,7 @@ module Herringbone
 
     # @param n [Integer, nil] byte count
     # @return [String] e.g. "512 B", "1.50 KB" or "12.3 MB" (binary units); "?" for nil
+    # @api private
     def self.human_bytes(n)
       return "?" unless n
       units = %w[B KB MB GB TB]
@@ -1710,8 +1735,7 @@ module Herringbone
     # @raise [FormatError] when the file is too small, lacks the magic bytes or has a corrupt footer
     # @raise [DecryptionError] when the footer is encrypted and cannot be decrypted
     def read_footer
-      @io.seek(0, IO::SEEK_END)
-      @file_size = @io.pos
+      @file_size = @io.size
       raise FormatError, "File too small to be Parquet (#{@file_size} bytes)" if @file_size < 12
       tail = read_at(@file_size - 8, 8)
       magic = tail.byteslice(4, 4)

data/lib/herringbone/io_buffer_support.rb

@@ -4,6 +4,8 @@ module Herringbone
   # IO::Buffer lets the decompressors copy bytes around without allocating a String per copy.
   # It still carries an "experimental" warning, which is silenced once here: the gem only uses
   # new/for/copy/get_string/free, and falls back to String operations where it is missing.
+  #
+  # @api private
   module IOBufferSupport
     # true when IO::Buffer has the methods the decompressors use and HERRINGBONE_NO_IO_BUFFER is
     # not set in the environment (which forces the String fallback)

data/lib/herringbone/reader/bloom_filters.rb

@@ -14,6 +14,7 @@ module Herringbone
     # @raise [FormatError] when the filter is truncated or its header cannot be decoded
     # @raise [DecryptionError] when the column is encrypted and its key was not given, or the
     #   filter does not decrypt
+    # @api private
     def bloom_filter(row_group_index, column)
       col = bloom_filter_column(column)
       rg = row_groups.fetch(row_group_index) { raise IndexError, "No row group #{row_group_index}" }

data/lib/herringbone/reader/chunk_copier.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module Herringbone
+  class Reader
+    # Internal (used by Redaction and Herringbone.combine): takes plaintext column chunks out of a
+    # file as they are stored, for Writer#write_row_group to copy into another file, and tells how
+    # a chunk that has to be encoded again should be compressed.
+    #
+    # @api private
+    class ChunkCopier
+      # Bytes read at a time when a page header runs past what was read of a chunk
+      READ_MORE = 64 * 1024
+      # Footer metadata that describes the columns, and goes stale when the schema changes
+      COLUMN_METADATA_KEYS = %w[ARROW:schema pandas].freeze
+
+      # @param reader [Reader] the file, whose IO the chunks are read from
+      def initialize(reader)
+        @reader = reader
+        @io = reader.io
+      end
+
+      # @param i [Integer] row group index
+      # @param col [Schema::Column] the column, in the file's schema
+      # @return [Writer::CopiedChunk] the chunk's pages, page index and bloom filter
+      # @raise [UnsupportedError] for an encrypted chunk or one stored in another file
+      # @raise [FormatError] when a page header is corrupt or the chunk overruns the file
+      def copy(i, col)
+        chunk = @reader.row_groups[i].columns.fetch(col.index)
+        meta = chunk.meta_data or raise UnsupportedError, "Column chunk without metadata (encrypted?)"
+        raise UnsupportedError, "Column chunks in external files are not supported" if chunk.file_path
+        column_index, offset_index = @reader.page_index(i, col)
+        start, bytes = chunk_bytes(col, meta, offset_index)
+        column_index &&= read_at(chunk.column_index_offset, chunk.column_index_length)
+        Writer::CopiedChunk.new(chunk, start, bytes, column_index, offset_index, bloom_bytes(i, col, meta))
+      end
+
+      # How a chunk encoded again follows the source chunk: the same codec (LZO cannot be written
+      # and becomes Snappy), and a bloom filter when the source chunk had one
+      #
+      # @param i [Integer] row group index
+      # @param col [Schema::Column] the column, in the file's schema
+      # @return [Array(Integer, Boolean), nil] codec id and whether to write a bloom filter, nil when
+      #   the chunk's metadata is not readable
+      def recode_settings(i, col)
+        meta = @reader.row_groups[i].columns.fetch(col.index).meta_data
+        return nil unless meta
+        [(meta.codec == Format::Codec::LZO) ? Format::Codec::SNAPPY : meta.codec, !meta.bloom_filter_offset.nil?]
+      end
+
+      # The encrypted columns of a file encrypted like this one, as the +columns:+ of its
+      # EncryptionConfiguration: those encrypted with the footer key as +:footer+, the others
+      # with their key and key metadata
+      #
+      # @param sources [Hash{String => Schema::Column}] the new file's column path => the column
+      #   here that holds its values
+      # @return [Hash{String => Symbol, Hash}] the encrypted ones of +sources+
+      # @raise [DecryptionError] when the key of an encrypted column was not given
+      def encrypted_columns(sources)
+        decryptor = @reader.decryptor or return {}
+        chunks = @reader.row_groups.first&.columns || []
+        sources.filter_map do |path, col|
+          chunk = chunks[col.index]
+          crypto = chunk&.crypto_metadata or next
+          with_column_key = crypto.encryption_with_column_key
+          next [path, :footer] unless with_column_key
+          key = decryptor.chunk_key(chunk, col.dotted_path)
+          key or raise DecryptionError, "The output is encrypted like the input, which needs the key of #{col.dotted_path}: " \
+            "pass it in decryption:, or pass encryption: for the output"
+          [path, {key: key, key_metadata: with_column_key.key_metadata}]
+        end.to_h
+      end
+
+      # The row group's sorting columns, pointed at the new file's columns. A column missing from
+      # +output_index+ ends the list, since the columns after it were only sorted within its runs.
+      #
+      # @param i [Integer] row group index
+      # @param output_index [Hash{Integer => Integer}] column index here => column index in the new
+      #   file, for the columns whose values the new file holds as they are
+      # @return [Array<Format::SortingColumn>, nil]
+      def sorting_columns(i, output_index)
+        out = (@reader.row_groups[i].sorting_columns || []).each_with_object([]) do |sc, acc|
+          j = output_index[sc.column_idx] or break acc
+          acc << Format::SortingColumn.new(column_idx: j, descending: sc.descending, nulls_first: sc.nulls_first)
+        end
+        out.empty? ? nil : out
+      end
+
+      # @param schema [Schema] the new file's schema
+      # @return [Hash{String => String, nil}] the file's key/value metadata, without the keys that
+      #   describe the columns when +schema+ is another
+      def metadata(schema)
+        (@reader.schema == schema) ? @reader.metadata : @reader.metadata.except(*COLUMN_METADATA_KEYS)
+      end
+
+      private
+
+      # The chunk's pages. They are walked header by header rather than trusting
+      # total_compressed_size, which some writers get wrong: copying too little breaks the
+      # chunk, and copying too much could carry over bytes of a neighbouring chunk.
+      #
+      # @param col [Schema::Column] the column
+      # @param meta [Format::ColumnMetaData] the chunk's metadata
+      # @param offset_index [Format::OffsetIndex, nil] the chunk's OffsetIndex, whose pages are
+      #   included even when num_values is reached before them
+      # @return [Array(Integer, String)] file offset of the first page, and the pages' bytes
+      # @raise [FormatError] when a page header is corrupt or the chunk overruns the file
+      def chunk_bytes(col, meta, offset_index)
+        start = meta.data_page_offset
+        dict = meta.dictionary_page_offset
+        start = dict if dict&.positive? && dict < start
+        last = offset_index&.page_locations&.last
+        min_end = last ? last.offset + last.compressed_page_size - start : 0
+        @io.seek(start)
+        buf = (@io.read(meta.total_compressed_size) || "").b
+        pos = 0
+        seen = 0
+        while seen < meta.num_values || pos < min_end
+          begin
+            header, body = Format::PageHeader.decode(buf, pos)
+          rescue Thrift::Error
+            raise FormatError, "Corrupt page header in #{col.dotted_path}" if read_more(buf, start, READ_MORE).zero?
+            retry
+          end
+          size = header.compressed_page_size
+          raise FormatError, "Negative page size in #{col.dotted_path}" if size.nil? || size.negative?
+          short = body + size - buf.bytesize
+          if short.positive? && read_more(buf, start, short) < short
+            raise FormatError, "Column #{col.dotted_path}: page overruns the file"
+          end
+          pos = body + size
+          seen += header.data_page_header&.num_values || header.data_page_header_v2&.num_values || 0
+        end
+        [start, (pos == buf.bytesize) ? buf : buf.byteslice(0, pos)]
+      end
+
+      # Appends up to +count+ bytes that follow +buf+ in the file
+      #
+      # @param buf [String] bytes read from +start+ on; appended to
+      # @param start [Integer] file offset of +buf+
+      # @param count [Integer] bytes wanted
+      # @return [Integer] bytes appended, 0 at the end of the file
+      def read_more(buf, start, count)
+        @io.seek(start + buf.bytesize)
+        more = @io.read(count)
+        return 0 if more.nil?
+        buf << more.b
+        more.bytesize
+      end
+
+      # @param offset [Integer, nil] file offset
+      # @param length [Integer, nil] byte count
+      # @return [String, nil] the bytes, nil when absent or cut off
+      def read_at(offset, length)
+        return nil unless offset && length&.positive?
+        @io.seek(offset)
+        bytes = @io.read(length)
+        (bytes&.bytesize == length) ? bytes.b : nil
+      end
+
+      # The chunk's bloom filter as stored. Filters written without a length (older writers) are
+      # decoded and encoded again. A filter that cannot be read is left out, which only costs
+      # pruning.
+      #
+      # @param i [Integer] row group index
+      # @param col [Schema::Column] the column
+      # @param meta [Format::ColumnMetaData] the chunk's metadata
+      # @return [String, nil] header and bitset
+      def bloom_bytes(i, col, meta)
+        return nil unless meta.bloom_filter_offset
+        return read_at(meta.bloom_filter_offset, meta.bloom_filter_length) if meta.bloom_filter_length
+        @reader.bloom_filter(i, col)&.encode
+      rescue FormatError
+        nil
+      end
+    end
+  end
+end

data/lib/herringbone/reader/column_chunk_reader.rb

@@ -13,6 +13,8 @@ module Herringbone
     #
     # The declared total_compressed_size of the chunk is not relied on (some old writers
     # under-report it): pages are read until the chunk's num_values have been seen.
+    #
+    # @api private
     class ColumnChunkReader
       # Shorthand for the physical type constants
       T = Format::Type
@@ -37,7 +39,7 @@ module Herringbone
       # @return [Proc, nil] converter of the last page returned by #next_page, nil when none
       attr_reader :page_converter
 
-      # @param io [IO, StringIO] the file, read with #seek and #read
+      # @param io [IO, StringIO, RestrictedReadableIO] the file, read with #seek and #read
       # @param chunk [Format::ColumnChunk] the chunk's footer entry
       # @param column [Schema::Column] the leaf column the chunk stores
       # @param converter [Proc, nil] physical value => Ruby value, applied to values and dictionaries
@@ -46,7 +48,7 @@ module Herringbone
       #   encrypted column
       # @raise [UnsupportedError] for chunks without metadata (encrypted) or stored in another file
       def initialize(io, chunk, column, converter: column.converter, lazy: false, crypto: nil)
-        @io = io
+        @io = RestrictedReadableIO.wrap(io)
         @chunk = chunk
         @column = column
         @meta = chunk.meta_data or raise UnsupportedError, "Column chunk without metadata (encrypted?)"

data/lib/herringbone/reader/column_cursor.rb

@@ -7,6 +7,8 @@ module Herringbone
     # entries of the requested rows (plus a small lookahead of levels for repeated columns)
     # become Ruby objects. A row starts at an entry with repetition level 0 and may continue
     # over any number of following pages.
+    #
+    # @api private
     class ColumnCursor
       # Levels decoded ahead at a time when looking for row starts in repeated columns
       LOOKAHEAD = 4096

data/lib/herringbone/reader/numo.rb

@@ -20,6 +20,8 @@ module Herringbone
     #   list<number>, all rows of the   2-D [rows, length] of the element type
     #     same length and no nulls
     #   anything else                   RObject of the values as: :rows returns
+    #
+    # @api private
     module NumoColumns
       # Whether the host is little-endian, so PLAIN bytes (always little-endian) can be handed to
       # Numo's from_binary, which reads native byte order.
@@ -224,6 +226,8 @@ module Herringbone
     # Hands out the next +k+ rows of a flat (non-repeated) numeric or boolean column as a
     # [Numo values, validity] pair: values has one slot per row (zero where the row is null),
     # validity is a Numo::Bit or nil when all rows are present.
+    #
+    # @api private
     class NumoCursor
       # @param chunk_reader [ColumnChunkReader] reader for the column's chunk in one row group
       # @param spec [NumoColumns::Spec] a :fixed spec with a bin_klass

data/lib/herringbone/reader/page_stream.rb

@@ -5,6 +5,8 @@ module Herringbone
     # Incremental decoders for the contents of one data page. The page bytes are decoded as they
     # are asked for, so a caller that takes a few hundred entries at a time never holds a whole
     # page's worth of levels or values as Ruby objects.
+    #
+    # @api private
     module PageStream
       # Values unpacked from a bit-packed run at a time (a multiple of 8)
       CHUNK = 1024

data/lib/herringbone/reader/scan.rb

@@ -16,6 +16,8 @@ module Herringbone
     # bloom filters; within a row group, pages are ruled out with the page index (ColumnIndex +
     # OffsetIndex), leaving ranges of rows to read; and every row that is read is checked, so
     # results are exact.
+    #
+    # @api private
     class Filter
       # One where: entry, resolved against the schema
       #

data/lib/herringbone/reader.rb

@@ -44,9 +44,16 @@ module Herringbone
     # @return [Format::FileMetaData] the file's FileMetaData (the decoded Thrift footer)
     attr_reader :file_metadata
 
-    # +io+ must support #seek and #read (a File opened with "rb", StringIO, Tempfile...)
+    # @return [RestrictedReadableIO] the IO the file is read from, as given to Reader.new, wrapped
+    attr_reader :io
+
+    # @return [DecryptionConfiguration, nil] the keys given as +decryption:+, nil without them
+    attr_reader :decryption
+
+    # +io+ must support #seek and #read (a File opened with "rb", StringIO, Tempfile...). It is
+    # read only through a RestrictedReadableIO wrapping it.
     #
-    # @param io [IO, StringIO] random-access source of the Parquet bytes; the caller closes it
+    # @param io [IO, StringIO, RestrictedReadableIO] random-access source of the Parquet bytes; the caller closes it
     # @param keys [Symbol, String] +:string+ or +:symbol+, the key type of row and struct Hashes
     # @param time_zone [String, Integer, Object, nil] zone timestamps are returned in (see the
     #   class docs); nil keeps them in UTC
@@ -74,7 +81,7 @@ module Herringbone
       @symbolize = keys == :symbol
       @zone_converter = zone_converter(time_zone)
       @decryption = decryption.nil? ? nil : DecryptionConfiguration.from(decryption)
-      @io = io
+      @io = RestrictedReadableIO.wrap(io)
       @file_metadata = read_footer
       @schema = Schema.from_elements(@file_metadata.schema)
     end
@@ -301,6 +308,7 @@ module Herringbone
     # @return [Array(Format::ColumnIndex, Format::OffsetIndex)] either element may be nil
     # @raise [ArgumentError] when +column+ does not name a leaf column
     # @raise [IndexError] when the row group does not exist
+    # @api private
     def page_index(row_group_index, column)
       column = @schema.column(column) unless column.is_a?(Schema::Column)
       raise ArgumentError, "No such leaf column" unless column
@@ -328,9 +336,10 @@ module Herringbone
     #   checked) and +:columns+
     def encryption = @decryptor&.describe(@schema, row_groups.first&.columns || [])
 
-    # Internal (used by Redaction): the keys and settings of an encrypted file
+    # Internal (used by Redaction and Combiner): the keys and settings of an encrypted file
     #
     # @return [Encryption::FileDecryptor, nil] nil for a file that is not encrypted
+    # @api private
     attr_reader :decryptor
 
     # Internal: the decryption of one column chunk's modules
@@ -339,6 +348,7 @@ module Herringbone
     # @param column [Schema::Column] leaf column
     # @return [Encryption::ModuleCrypto, nil] nil when the chunk is not encrypted
     # @raise [DecryptionError] when the chunk is encrypted and its key was not given
+    # @api private
     def chunk_crypto(row_group_index, column)
       return nil unless @decryptor
       rg = row_groups.fetch(row_group_index)
@@ -354,6 +364,7 @@ module Herringbone
     # @option options [Boolean] :lazy leave non-dictionary values physical
     # @return [ColumnChunkReader]
     # @raise [DecryptionError] when the chunk is encrypted and its key was not given
+    # @api private
     def chunk_reader(row_group_index, column, **options)
       chunk = row_groups.fetch(row_group_index).columns.fetch(column.index)
       ColumnChunkReader.new(@io, chunk, column, crypto: chunk_crypto(row_group_index, column), **options)
@@ -656,8 +667,7 @@ module Herringbone
     # @raise [FormatError] when the file is too short, lacks the magic or the footer is corrupt
     # @raise [DecryptionError] when the footer is encrypted and cannot be decrypted
     def read_footer
-      @io.seek(0, IO::SEEK_END)
-      size = @io.pos
+      size = @io.size
       raise FormatError, "File too small to be Parquet (#{size} bytes)" if size < 12
       @io.seek(size - 8)
       tail = @io.read(8)
@@ -679,6 +689,8 @@ module Herringbone
 
     # Rebuilds nested values of one top-level field from the levels of its leaf columns
     # (the "record assembly" half of the Dremel algorithm).
+    #
+    # @api private
     class Assembler
       # @param field [Schema::Field] top-level field to assemble
       # @param symbolize [Boolean] whether struct Hashes are keyed by Symbol instead of String
@@ -860,6 +872,7 @@ require_relative "reader/bloom_filters"
 
 module Herringbone
   class Reader
+    autoload :ChunkCopier, File.expand_path("reader/chunk_copier", __dir__)
     autoload :Filter, File.expand_path("reader/scan", __dir__)
     autoload :NumoColumns, File.expand_path("reader/numo", __dir__)
     autoload :NumoCursor, File.expand_path("reader/numo", __dir__)