herringbone 0.5.0 → 0.6.0

data/lib/herringbone/reader/column_chunk_reader.rb

@@ -42,8 +42,10 @@ module Herringbone
       # @param column [Schema::Column] the leaf column the chunk stores
       # @param converter [Proc, nil] physical value => Ruby value, applied to values and dictionaries
       # @param lazy [Boolean] leave non-dictionary values physical in #next_page (see #page_converter)
+      # @param crypto [Encryption::ModuleCrypto, nil] decryption of the chunk's pages, for an
+      #   encrypted column
       # @raise [UnsupportedError] for chunks without metadata (encrypted) or stored in another file
-      def initialize(io, chunk, column, converter: column.converter, lazy: false)
+      def initialize(io, chunk, column, converter: column.converter, lazy: false, crypto: nil)
         @io = io
         @chunk = chunk
         @column = column
@@ -59,6 +61,11 @@ module Herringbone
         start = dict if dict&.positive? && dict < start
         @pos = start
         @start = start
+        @crypto = crypto
+        # An encrypted header's AAD depends on whether it is the dictionary page's, which only its
+        # position tells before it is decrypted
+        @dictionary_offset = (start < @meta.data_page_offset) ? start : nil
+        @ordinal = 0 # data pages read so far, part of an encrypted data page's AAD
         @locations = nil # OffsetIndex page locations, when jumping between pages
         @page_number = nil
         @total = @meta.num_values
@@ -154,6 +161,7 @@ module Herringbone
         load_dictionary
         @pos = @locations.fetch(index).offset
         @page_number = index
+        @ordinal = index
       end
 
       # Whether all of the chunk's values have been returned
@@ -196,7 +204,9 @@ module Herringbone
       #
       # @return [Array(Format::PageHeader, String)] the header and the (still compressed) body
       # @raise [FormatError] for a negative page size or a page that overruns the file
+      # @raise [DecryptionError] when an encrypted page does not decrypt
       def read_page
+        return read_encrypted_page if @crypto
         # With an OffsetIndex the page's size (header included) is known, so read exactly that
         loc = @page_number && @locations[@page_number]
         exact = loc && loc.offset == @pos && loc.compressed_page_size.positive?
@@ -222,6 +232,50 @@ module Herringbone
         [header, body]
       end
 
+      # Like #read_page, for an encrypted column: the header and the body are each an encrypted
+      # module, the header's starting with its length
+      #
+      # @return [Array(Format::PageHeader, String)] the header and the decrypted (still compressed) body
+      # @raise [FormatError] for a page that overruns the file
+      # @raise [DecryptionError] when the header or the body does not decrypt
+      def read_encrypted_page
+        loc = @page_number && @locations[@page_number]
+        exact = loc && loc.offset == @pos && loc.compressed_page_size.positive?
+        dictionary = @pos == @dictionary_offset
+        ordinal = dictionary ? nil : @ordinal
+        mod = read_module(@pos, exact ? loc.compressed_page_size : HEADER_GUESS)
+        type = dictionary ? Encryption::DICTIONARY_PAGE_HEADER : Encryption::DATA_PAGE_HEADER
+        header = Format::PageHeader.decode(@crypto.decrypt(type, mod, ordinal)).first
+        size = header.compressed_page_size
+        raise FormatError, "Negative page size in #{@column.dotted_path}" if size.nil? || size.negative?
+        buf, off = window(@pos + mod.bytesize, size + (exact ? 0 : READ_AHEAD), size)
+        if buf.bytesize - off < size
+          raise FormatError, "Column #{@column.dotted_path}: page overruns the file (read #{@seen} of #{@total} values)"
+        end
+        type = (header.type == Format::PageType::DICTIONARY_PAGE) ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE
+        body = @crypto.decrypt(type, buf.byteslice(off, size), ordinal)
+        @pos += mod.bytesize + size
+        @ordinal += 1 unless dictionary
+        [header, body]
+      end
+
+      # An encrypted module (length prefix included) at +pos+
+      #
+      # @param pos [Integer] file offset of the module
+      # @param guess [Integer] bytes to read when the buffer does not hold the length prefix
+      # @return [String] the module
+      # @raise [FormatError] when the module overruns the file
+      def read_module(pos, guess)
+        buf, off = window(pos, guess, 4)
+        len = buf.bytesize - off >= 4 && buf.byteslice(off, 4).unpack1("V")
+        raise FormatError, "Column #{@column.dotted_path}: encrypted page header overruns the file" unless len
+        buf, off = window(pos, 4 + len + READ_AHEAD, 4 + len)
+        if buf.bytesize - off < 4 + len
+          raise FormatError, "Column #{@column.dotted_path}: encrypted page header overruns the file"
+        end
+        buf.byteslice(off, 4 + len)
+      end
+
       # Returns [buffer, offset] where buffer[offset..] holds at least +need+ bytes from file
       # position +pos+ (fewer only at EOF), reading +len+ bytes when the buffer does not cover it.
       #

data/lib/herringbone/reader.rb

@@ -21,9 +21,16 @@ module Herringbone
   #              seconds as an Integer), a timezone object Time#getlocal accepts (e.g. a
   #              TZInfo::Timezone), or anything responding to #at such as an
   #              ActiveSupport::TimeZone (Time.zone), which yields ActiveSupport::TimeWithZone.
+  #   decryption: keys for a file written with Parquet modular encryption:
+  #              { footer_key: "...", columns: { "ssn" => "..." }, aad_prefix: "..." }, and/or
+  #              keys: ->(key_metadata) { ... } (or a Hash) to look keys up by the key metadata
+  #              stored in the file. Keys are 16, 24 or 32-byte Strings. Plaintext columns of a
+  #              file with a plaintext footer can be read without keys.
   class Reader
     # The 4 bytes a Parquet file starts and ends with
     MAGIC = "PAR1"
+    # The 4 bytes a Parquet file with an encrypted footer starts and ends with
+    ENCRYPTED_MAGIC = "PARE"
     # Rows per batch in #each_batch when no size is given
     DEFAULT_BATCH_SIZE = 1024
     # Accepted values of the +keys:+ option
@@ -43,9 +50,20 @@ module Herringbone
     # @param keys [Symbol, String] +:string+ or +:symbol+, the key type of row and struct Hashes
     # @param time_zone [String, Integer, Object, nil] zone timestamps are returned in (see the
     #   class docs); nil keeps them in UTC
-    # @raise [ArgumentError] when +io+ cannot seek and read, or +keys+ / +time_zone+ are invalid
+    # @param decryption [DecryptionConfiguration, Hash{Symbol => Object}, #call, nil] keys for an
+    #   encrypted file (see the class docs); a callable is used as +keys:+
+    # @option decryption [String] :footer_key key of the footer (and of the columns encrypted with it)
+    # @option decryption [Hash{String => String}] :columns column path or field name => key
+    # @option decryption [#call, Hash{String => String}] :keys key metadata => key, for the keys not
+    #   given above; returns nil for a key that is not available
+    # @option decryption [String] :aad_prefix the file's AAD prefix, when it does not store it (or
+    #   to check the stored one)
+    # @raise [ArgumentError] when +io+ cannot seek and read, or +keys+ / +time_zone+ /
+    #   +decryption+ are invalid
     # @raise [FormatError] when the footer is missing or cannot be decoded
-    def initialize(io, keys: :string, time_zone: nil)
+    # @raise [DecryptionError] when the footer is encrypted and cannot be decrypted, or its
+    #   signature or a column's metadata does not check out
+    def initialize(io, keys: :string, time_zone: nil, decryption: nil)
       unless io.respond_to?(:seek) && io.respond_to?(:read)
         raise ArgumentError, "Herringbone::Reader expects an IO that supports #seek and #read " \
           "(e.g. File.open(path, \"rb\")), got #{io.is_a?(String) ? "a String" : io.class}" \
@@ -55,6 +73,7 @@ module Herringbone
       raise ArgumentError, "keys: must be :string or :symbol, got #{keys.inspect}" unless KEY_MODES.include?(keys)
       @symbolize = keys == :symbol
       @zone_converter = zone_converter(time_zone)
+      @decryption = decryption.nil? ? nil : DecryptionConfiguration.from(decryption)
       @io = io
       @file_metadata = read_footer
       @schema = Schema.from_elements(@file_metadata.schema)
@@ -162,7 +181,7 @@ module Herringbone
         partial = ranges != [[0, rg.num_rows]]
         cursors = fields.map do |f|
           f.leaves.map do |col|
-            reader = ColumnChunkReader.new(@io, rg.columns.fetch(col.index), col, converter: converters[col.index], lazy: true)
+            reader = chunk_reader(rg_index, col, converter: converters[col.index], lazy: true)
             reader.locations = page_index(rg_index, col)[1]&.page_locations if partial
             [col.index, ColumnCursor.new(reader)]
           end
@@ -288,11 +307,58 @@ module Herringbone
       @page_indexes ||= {}
       @page_indexes[[row_group_index, column.index]] ||= begin
         chunk = row_groups.fetch(row_group_index).columns.fetch(column.index)
-        [read_struct(Format::ColumnIndex, chunk.column_index_offset, chunk.column_index_length),
-          read_struct(Format::OffsetIndex, chunk.offset_index_offset, chunk.offset_index_length)]
+        crypto = (chunk.column_index_offset || chunk.offset_index_offset) && chunk_crypto(row_group_index, column)
+        [read_struct(Format::ColumnIndex, chunk.column_index_offset, chunk.column_index_length, crypto, crypto && Encryption::COLUMN_INDEX),
+          read_struct(Format::OffsetIndex, chunk.offset_index_offset, chunk.offset_index_length, crypto, crypto && Encryption::OFFSET_INDEX)]
       end
     end
 
+    # How the file is encrypted, without needing its keys; nil for a file that is not encrypted.
+    # +columns+ lists the encrypted columns (as the first row group has them), with the key
+    # metadata of their own key, if any, and whether their key is available.
+    #
+    #   reader.encryption
+    #   # => { algorithm: :aes_gcm, footer: :encrypted, footer_key_metadata: "kf", aad_prefix: nil,
+    #   #      supply_aad_prefix: false, footer_verified: false,
+    #   #      columns: { "ssn" => { key: :column, key_metadata: "kc1", readable: true } } }
+    #
+    # @return [Hash{Symbol => Object}, nil] +:algorithm+ (+:aes_gcm+ or +:aes_gcm_ctr+), +:footer+
+    #   (+:encrypted+ or +:plaintext+), +:footer_key_metadata+, +:aad_prefix+ (when stored),
+    #   +:supply_aad_prefix+, +:footer_verified+ (whether a plaintext footer's signature was
+    #   checked) and +:columns+
+    def encryption = @decryptor&.describe(@schema, row_groups.first&.columns || [])
+
+    # Internal (used by Redaction): the keys and settings of an encrypted file
+    #
+    # @return [Encryption::FileDecryptor, nil] nil for a file that is not encrypted
+    attr_reader :decryptor
+
+    # Internal: the decryption of one column chunk's modules
+    #
+    # @param row_group_index [Integer] position of the row group in the footer
+    # @param column [Schema::Column] leaf column
+    # @return [Encryption::ModuleCrypto, nil] nil when the chunk is not encrypted
+    # @raise [DecryptionError] when the chunk is encrypted and its key was not given
+    def chunk_crypto(row_group_index, column)
+      return nil unless @decryptor
+      rg = row_groups.fetch(row_group_index)
+      @decryptor.chunk(row_group_index, rg.ordinal, column, rg.columns.fetch(column.index))
+    end
+
+    # Internal: a ColumnChunkReader for a leaf column of a row group, decrypting when needed
+    #
+    # @param row_group_index [Integer] position of the row group in the footer
+    # @param column [Schema::Column] leaf column
+    # @param options [Hash{Symbol => Object}] passed to ColumnChunkReader.new
+    # @option options [Proc, nil] :converter physical value => Ruby value
+    # @option options [Boolean] :lazy leave non-dictionary values physical
+    # @return [ColumnChunkReader]
+    # @raise [DecryptionError] when the chunk is encrypted and its key was not given
+    def chunk_reader(row_group_index, column, **options)
+      chunk = row_groups.fetch(row_group_index).columns.fetch(column.index)
+      ColumnChunkReader.new(@io, chunk, column, crypto: chunk_crypto(row_group_index, column), **options)
+    end
+
     # Short summary for the console, without the schema
     #
     # @return [String] row count, row group count and the writer's created_by
@@ -348,7 +414,7 @@ module Herringbone
         rg = row_groups[rg_index]
         partial = ranges != [[0, rg.num_rows]]
         open = lambda do |col|
-          reader = ColumnChunkReader.new(@io, rg.columns.fetch(col.index), col, converter: converters[col.index], lazy: true)
+          reader = chunk_reader(rg_index, col, converter: converters[col.index], lazy: true)
           reader.locations = page_index(rg_index, col)[1]&.page_locations if partial
           reader
         end
@@ -458,14 +524,18 @@ module Herringbone
     # @param klass [Class] Format struct class to decode with (responds to .decode)
     # @param offset [Integer, nil] file offset of the struct
     # @param length [Integer, nil] byte length of the struct
+    # @param crypto [Encryption::ModuleCrypto, nil] decryption of the chunk's modules
+    # @param type [Integer, nil] the struct's module type, when encrypted
     # @return [Object, nil] the decoded +klass+ instance, or nil when absent, truncated or corrupt
-    def read_struct(klass, offset, length)
+    # @raise [DecryptionError] when an encrypted struct does not decrypt
+    def read_struct(klass, offset, length, crypto = nil, type = nil)
       return nil unless offset && length&.positive?
       @io.seek(offset)
       bytes = @io.read(length)
       return nil unless bytes&.bytesize == length
+      bytes = crypto.decrypt(type, bytes.b) if crypto
       klass.decode(bytes.b).first
-    rescue Thrift::Error
+    rescue Thrift::Error, FormatError
       nil # a damaged index only means pages cannot be skipped
     end
 
@@ -579,22 +649,30 @@ module Herringbone
     # Reads and decodes the footer: the FileMetaData Thrift struct, its 4-byte little-endian
     # length and the closing magic.
     #
+    # In an encrypted file the footer is decrypted (or its signature checked), and so is the
+    # metadata of the columns whose key is available.
+    #
     # @return [Format::FileMetaData] the decoded footer
     # @raise [FormatError] when the file is too short, lacks the magic or the footer is corrupt
-    # @raise [UnsupportedError] for an encrypted file (+PARE+ magic)
+    # @raise [DecryptionError] when the footer is encrypted and cannot be decrypted
     def read_footer
       @io.seek(0, IO::SEEK_END)
       size = @io.pos
       raise FormatError, "File too small to be Parquet (#{size} bytes)" if size < 12
       @io.seek(size - 8)
       tail = @io.read(8)
-      raise UnsupportedError, "Encrypted Parquet files are not supported" if tail.byteslice(4, 4) == "PARE"
-      raise FormatError, "Missing PAR1 footer magic" unless tail.byteslice(4, 4) == MAGIC
+      magic = tail.byteslice(4, 4)
+      raise FormatError, "Missing PAR1 footer magic" unless magic == MAGIC || magic == ENCRYPTED_MAGIC
       footer_len = tail.unpack1("V")
       raise FormatError, "Footer length #{footer_len} exceeds file size" if footer_len + 12 > size
       @io.seek(size - 8 - footer_len)
-      footer = @io.read(footer_len)
-      Format::FileMetaData.decode(footer).first
+      footer = @io.read(footer_len).b
+      if magic == MAGIC
+        meta = Format::FileMetaData.decode(footer).first
+        return meta unless meta.encryption_algorithm
+      end
+      meta, @decryptor = Encryption.read_footer(footer, magic, @decryption)
+      meta
     rescue Thrift::Error => e
       raise FormatError, "Corrupt file metadata: #{e.message}"
     end
@@ -778,5 +856,12 @@ end
 require_relative "reader/page_stream"
 require_relative "reader/column_chunk_reader"
 require_relative "reader/column_cursor"
-require_relative "reader/scan"
-require_relative "reader/numo"
+require_relative "reader/bloom_filters"
+
+module Herringbone
+  class Reader
+    autoload :Filter, File.expand_path("reader/scan", __dir__)
+    autoload :NumoColumns, File.expand_path("reader/numo", __dir__)
+    autoload :NumoCursor, File.expand_path("reader/numo", __dir__)
+  end
+end

data/lib/herringbone/redaction/rewriter.rb

@@ -34,11 +34,13 @@ module Herringbone
 
       # @param redaction [Redaction] the statements and drops to apply
       # @param io [IO, StringIO] the input file, read with #seek and #read
+      # @param decryption [DecryptionConfiguration, Hash{Symbol => Object}, nil] keys of an encrypted input, see Reader.new
       # @raise [ArgumentError] when a statement or drop does not fit the file's schema
       # @raise [FormatError] when the footer cannot be read
-      def initialize(redaction, io)
+      # @raise [DecryptionError] when the footer is encrypted and cannot be decrypted
+      def initialize(redaction, io, decryption: nil)
         @io = io
-        @reader = Reader.new(io)
+        @reader = Reader.new(io, decryption: decryption)
         @schema = @reader.schema
         @statements = redaction.statements
         @filters = @statements.map { |s| s.where && Reader::Filter.new(@schema, s.where) }
@@ -67,6 +69,7 @@ module Herringbone
       # @param output [IO, #write] destination
       # @param options [Hash{Symbol => Object}] Writer options for re-encoded chunks, and +metadata:+
       # @option options [Symbol] :compression (codec of each source chunk) codec for re-encoded chunks
+      # @option options [Integer, nil] :compression_level (nil) level for that codec, see Writer
       # @option options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
       #   columns whose re-encoded chunks get a bloom filter, besides those whose source chunk had one
       # @option options [Hash{String => String}] :metadata (the input's) footer key/value metadata
@@ -75,14 +78,17 @@ module Herringbone
       # @option options [Integer] :data_page_version (1) 1 or 2
       # @option options [Boolean, Array<String>] :dictionary (true) see Writer
       # @option options [Hash{String => Symbol}] :encodings ({}) see Writer
+      # @option options [EncryptionConfiguration, Hash{Symbol => Object}, false] :encryption (as the input) see Writer
       # @return [Report]
       # @raise [ArgumentError] for +row_group_bytes:+ / +row_group_rows:+ or an invalid writer option
       # @raise [EncodeError] when a replacement value cannot be written
+      # @raise [DecryptionError] when the output is to be encrypted like the input but a key is missing
       def apply(output, **options)
         bad = options.keys & ROW_GROUP_OPTIONS
         raise ArgumentError, "#{bad.join(", ")}: a redaction keeps the row groups of the input" unless bad.empty?
         @keep_codecs = !options.key?(:compression)
         options = {metadata: copied_metadata}.merge(options)
+        options[:encryption] = input_encryption unless options.key?(:encryption)
         writer = Writer.new(output, @output_schema, row_group_bytes: 1 << 62, **options)
         @report = Report.new(rows_read: 0, rows_deleted: 0, rows_changed: 0, row_groups: {copied: 0, rewritten: 0})
         begin
@@ -232,18 +238,25 @@ module Herringbone
         @schema.fields.to_h { |f| [f.name, @data[f.name][r]] }
       end
 
-      # Tier 1: every kept column chunk is copied as it is
+      # Tier 1: every kept column chunk is copied as it is, except those encrypted in the input or
+      # the output, which are encoded again
       #
       # @param i [Integer] row group index
       # @param writer [Writer] the output
       # @return [void]
       def copy_row_group(i, writer)
+        unless encrypted_columns(i, writer).empty?
+          rewrite_columns(i, writer, [])
+          @report.row_groups[:copied] += 1
+          return
+        end
         copies = @output_columns.each_with_index.to_h { |col, j| [j, copied_chunk(i, col)] }
         writer.write_row_group(rows(i), copies: copies, sorting_columns: sorting_columns(i, []))
         @report.row_groups[:copied] += 1
       end
 
-      # Tier 2: the chunks of the changed leaf columns are encoded again, the rest are copied
+      # Tier 2: the chunks of the changed leaf columns (and of encrypted ones) are encoded again,
+      # the rest are copied
       #
       # @param i [Integer] row group index
       # @param writer [Writer] the output
@@ -251,12 +264,15 @@ module Herringbone
       # @return [void]
       def rewrite_columns(i, writer, changed)
         rewritten = changed.map { |t| t.column.index }
-        changed.map { |t| t.path.first }.uniq.each { |name| writer.buffer_field(name, @data[name]) }
+        encoded = rewritten | encrypted_columns(i, writer)
+        names = encoded.map { |index| @schema.columns[index].path.first }.uniq
+        load(i, names)
+        names.each { |name| writer.buffer_field(name, @data[name]) }
         copies = {}
         codecs = {}
         blooms = {}
         @output_columns.each_with_index do |col, j|
-          if rewritten.include?(col.index)
+          if encoded.include?(col.index)
             chunk_settings(i, col, j, codecs, blooms)
           else
             copies[j] = copied_chunk(i, col)
@@ -264,7 +280,43 @@ module Herringbone
         end
         writer.write_row_group(rows(i), copies: copies, codecs: codecs, bloom_filters: blooms,
           sorting_columns: sorting_columns(i, rewritten))
-        @report.row_groups[:rewritten] += 1
+        @report.row_groups[:rewritten] += 1 unless changed.empty?
+      end
+
+      # Kept columns whose chunk in row group +i+ cannot be copied, because it is encrypted in the
+      # input (its AAD names the file and the chunk's place in it) or is to be encrypted
+      #
+      # @param i [Integer] row group index
+      # @param writer [Writer] the output
+      # @return [Array<Integer>] input column indexes
+      def encrypted_columns(i, writer)
+        chunks = @reader.row_groups[i].columns
+        @output_columns.each_with_index.filter_map do |col, j|
+          col.index if chunks.fetch(col.index).crypto_metadata || writer.encrypted_column?(j)
+        end
+      end
+
+      # The Writer +encryption:+ option that encrypts the output like the input: the same
+      # algorithm, footer mode, AAD prefix and keys, for the columns that are kept
+      #
+      # @return [EncryptionConfiguration, nil] nil for a plaintext input
+      # @raise [DecryptionError] when a key of the input was not given
+      def input_encryption
+        decryptor = @reader.decryptor or return nil
+        chunks = @reader.row_groups.first&.columns || []
+        columns = @output_columns.each_with_index.filter_map do |col, j|
+          chunk = chunks[col.index]
+          crypto = chunk&.crypto_metadata or next
+          path = @output_schema.columns[j].dotted_path
+          with_column_key = crypto.encryption_with_column_key
+          next [path, :footer] unless with_column_key
+          key = decryptor.chunk_key(chunk, col.dotted_path)
+          key or raise DecryptionError, "The output is encrypted like the input, which needs the key of #{col.dotted_path}: " \
+            "pass it in decryption:, or pass encryption: for the output"
+          [path, {key: key, key_metadata: with_column_key.key_metadata}]
+        end
+        uniform = !columns.empty? && columns.size == @output_columns.size && columns.all? { |_, v| v == :footer }
+        decryptor.writer_settings(uniform ? nil : columns.to_h)
       end
 
       # Tier 3: the remaining rows are encoded again, every column of them

data/lib/herringbone/redaction.rb

@@ -198,11 +198,13 @@ module Herringbone
     # the columns the conditions name are read for the rest.
     #
     # @param input_io [IO, StringIO] the Parquet file, read with #seek and #read
+    # @param decryption [DecryptionConfiguration, Hash{Symbol => Object}, nil] keys of an encrypted file, see Reader.new
     # @return [Boolean]
     # @raise [ArgumentError] when the redaction does not fit the file's schema
-    def affects?(input_io)
+    # @raise [DecryptionError] when the file is encrypted and a key it needs was not given
+    def affects?(input_io, decryption: nil)
       check_scopes!
-      Rewriter.new(self, input_io).affects?
+      Rewriter.new(self, input_io, decryption: decryption).affects?
     end
 
     # Writes the redacted copy of +input_io+ to +output_io+. The redaction is checked against the
@@ -210,8 +212,14 @@ module Herringbone
     # verb raise before anything is written. The output is left unfinished (no footer) when an
     # error happens later, for instance a block returning a value its column cannot store.
     #
+    # An encrypted input (opened with +decryption:+) is written encrypted the same way: same
+    # algorithm, footer mode, AAD prefix, keys and key metadata. +encryption:+ (see Writer) writes
+    # it with other settings, and +encryption: false+ writes a plaintext file. Column chunks that
+    # are encrypted in either file are re-encoded rather than copied.
+    #
     # @param input_io [IO, StringIO] the Parquet file, read with #seek and #read; not closed
     # @param output_io [IO, #write] destination, written sequentially; not closed
+    # @param decryption [DecryptionConfiguration, Hash{Symbol => Object}, nil] keys of an encrypted input, see Reader.new
     # @param writer_options [Hash{Symbol => Object}] Writer options for the re-encoded column chunks
     #   (+compression:+, +bloom_filters:+, +page_rows:+, +dictionary:+...), and +metadata:+ to
     #   replace the footer key/value metadata instead of copying it
@@ -224,13 +232,16 @@ module Herringbone
     # @option writer_options [Integer] :data_page_version (1) 1 or 2
     # @option writer_options [Boolean, Array<String>] :dictionary (true) see Writer
     # @option writer_options [Hash{String => Symbol}] :encodings ({}) see Writer
+    # @option writer_options [EncryptionConfiguration, Hash{Symbol => Object}, false] :encryption (as the input) see Writer;
+    #   false for a plaintext output
     # @return [Report] what was done
     # @raise [ArgumentError] when the redaction does not fit the file's schema, or for a writer
     #   option that does not apply (+row_group_bytes:+, +row_group_rows:+)
     # @raise [EncodeError] when a replacement value cannot be written to its column
-    def apply(input_io, output_io, **writer_options)
+    # @raise [DecryptionError] when the input is encrypted and a key it needs was not given
+    def apply(input_io, output_io, decryption: nil, **writer_options)
       check_scopes!
-      Rewriter.new(self, input_io).apply(output_io, **writer_options)
+      Rewriter.new(self, input_io, decryption: decryption).apply(output_io, **writer_options)
     end
 
     # Short summary for the console: one clause per statement, naming the columns but not the
@@ -284,4 +295,8 @@ module Herringbone
   end
 end
 
-require_relative "redaction/rewriter"
+module Herringbone
+  class Redaction
+    autoload :Rewriter, File.expand_path("redaction/rewriter", __dir__)
+  end
+end

data/lib/herringbone/schema.rb

@@ -844,3 +844,5 @@ module Herringbone
     end
   end
 end
+
+require_relative "active_record"

data/lib/herringbone/simple_writer.rb

@@ -24,6 +24,14 @@ module Herringbone
   # holds both numbers and text:
   #
   #   Herringbone::SimpleWriter.new(io) { |s| s.string :code }
+  #
+  # #encrypt! encrypts the file with one key (see Key):
+  #
+  #   Herringbone::SimpleWriter.open(file) do |sw|
+  #     sw.encrypt!(key: ENV["PARQUET_KEY"]) # the key's hex, or a Herringbone::Key
+  #     sw.headers!(:id, :name)
+  #     sw << [1, "John"]
+  #   end
   class SimpleWriter
     # Opens a writer, yields it and closes it, finishing the file. If the block raises, the file is
     # left unfinished (no footer), as with Writer.open. To declare columns, use #initialize and #close.
@@ -31,6 +39,7 @@ module Herringbone
     # @param io [IO, #write] destination; written sequentially, never closed
     # @param options [Hash{Symbol => Object}] Writer options (compression:, row_group_bytes:...)
     # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer, nil] :compression_level (nil) level for :zstd, :gzip or :brotli
     # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
     # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
     #   (other Writer options are passed on as well)
@@ -57,6 +66,7 @@ module Herringbone
     # @param io [IO, #write] destination; nothing is written to it until the column types are known
     # @param options [Hash{Symbol => Object}] Writer options (compression:, row_group_bytes:...)
     # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer, nil] :compression_level (nil) level for :zstd, :gzip or :brotli
     # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
     # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
     #   (other Writer options are passed on as well)
@@ -68,6 +78,7 @@ module Herringbone
     def initialize(io, **options, &overrides)
       Schema::Builder.check_block!(overrides, "Herringbone::SimpleWriter.new(io) { |s| s.string :code }")
       @headers = nil
+      @encrypted = !!options[:encryption]
       @overrides = overrides
       fix = "Herringbone::SimpleWriter.new(io) { |s| s.%s }"
       @writer = InferringWriter.new(io, fix: fix, **options) { |sample| schema_for(sample) }
@@ -89,6 +100,23 @@ module Herringbone
       self
     end
 
+    # Encrypts the file with one key, the way most Parquet readers can decrypt with that key (see
+    # EncryptionConfiguration.simple). Must be called before the first row. For a new key, pass
+    # +key: Herringbone::Key.generate+ and keep it (+key.hex+): the file can't be read without it.
+    #
+    # @param key [Key, String] a key, or its hex (32 or 64 digits; raw bytes go through Key.new)
+    # @return [Key] the key the file is encrypted with
+    # @raise [ArgumentError] after the first row, when called twice or after +encryption:+ was
+    #   given, for a String that is not the hex of a key, or a 192-bit key
+    def encrypt!(key:)
+      raise ArgumentError, "encrypt! must be called before the first row" if rows_written.positive?
+      raise ArgumentError, "The file is already encrypted (encrypt! or encryption:)" if @encrypted
+      key = Key.from(key)
+      @writer.encryption = EncryptionConfiguration.simple(key)
+      @encrypted = true
+      key
+    end
+
     # Appends a row: an Array with one value per header, in header order, or a Hash keyed by
     # header (String or Symbol keys; missing columns are nulls).
     #

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end