herringbone 0.4.0 → 0.6.0

data/lib/herringbone/simple_writer.rb

@@ -23,7 +23,15 @@ module Herringbone
   # Columns declared in a block (Schema::Builder DSL) replace inferred ones, e.g. for a column that
   # holds both numbers and text:
   #
-  #   Herringbone::SimpleWriter.new(io) { string :code }
+  #   Herringbone::SimpleWriter.new(io) { |s| s.string :code }
+  #
+  # #encrypt! encrypts the file with one key (see Key):
+  #
+  #   Herringbone::SimpleWriter.open(file) do |sw|
+  #     sw.encrypt!(key: ENV["PARQUET_KEY"]) # the key's hex, or a Herringbone::Key
+  #     sw.headers!(:id, :name)
+  #     sw << [1, "John"]
+  #   end
   class SimpleWriter
     # Opens a writer, yields it and closes it, finishing the file. If the block raises, the file is
     # left unfinished (no footer), as with Writer.open. To declare columns, use #initialize and #close.
@@ -31,6 +39,7 @@ module Herringbone
     # @param io [IO, #write] destination; written sequentially, never closed
     # @param options [Hash{Symbol => Object}] Writer options (compression:, row_group_bytes:...)
     # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer, nil] :compression_level (nil) level for :zstd, :gzip or :brotli
     # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
     # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
     #   (other Writer options are passed on as well)
@@ -57,16 +66,21 @@ module Herringbone
     # @param io [IO, #write] destination; nothing is written to it until the column types are known
     # @param options [Hash{Symbol => Object}] Writer options (compression:, row_group_bytes:...)
     # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer, nil] :compression_level (nil) level for :zstd, :gzip or :brotli
     # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
     # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
     #   (other Writer options are passed on as well)
-    # @yield optional block evaluated with +instance_eval+ on a Schema::Builder, declaring columns
-    #   (named in #headers!) that replace inferred ones
+    # @yield [s] optional, declares columns (named in #headers!) that replace inferred ones
+    # @yieldparam s [Schema::Builder] the builder to declare columns on
+    # @yieldreturn [void]
     # @raise [MissingCodecError] when the codec's optional gem is not loaded
+    # @raise [ArgumentError] when the block takes no parameter
     def initialize(io, **options, &overrides)
+      Schema::Builder.check_block!(overrides, "Herringbone::SimpleWriter.new(io) { |s| s.string :code }")
       @headers = nil
+      @encrypted = !!options[:encryption]
       @overrides = overrides
-      fix = "Herringbone::SimpleWriter.new(io) { %s }"
+      fix = "Herringbone::SimpleWriter.new(io) { |s| s.%s }"
       @writer = InferringWriter.new(io, fix: fix, **options) { |sample| schema_for(sample) }
     end
 
@@ -86,6 +100,23 @@ module Herringbone
       self
     end
 
+    # Encrypts the file with one key, the way most Parquet readers can decrypt with that key (see
+    # EncryptionConfiguration.simple). Must be called before the first row. For a new key, pass
+    # +key: Herringbone::Key.generate+ and keep it (+key.hex+): the file can't be read without it.
+    #
+    # @param key [Key, String] a key, or its hex (32 or 64 digits; raw bytes go through Key.new)
+    # @return [Key] the key the file is encrypted with
+    # @raise [ArgumentError] after the first row, when called twice or after +encryption:+ was
+    #   given, for a String that is not the hex of a key, or a 192-bit key
+    def encrypt!(key:)
+      raise ArgumentError, "encrypt! must be called before the first row" if rows_written.positive?
+      raise ArgumentError, "The file is already encrypted (encrypt! or encryption:)" if @encrypted
+      key = Key.from(key)
+      @writer.encryption = EncryptionConfiguration.simple(key)
+      @encrypted = true
+      key
+    end
+
     # Appends a row: an Array with one value per header, in header order, or a Hash keyed by
     # header (String or Symbol keys; missing columns are nulls).
     #
@@ -117,6 +148,13 @@ module Herringbone
     # @return [void]
     def abort = @writer.abort
 
+    # Short summary for the console, without the rows
+    #
+    # @return [String] the number of columns and the underlying writer's summary
+    def inspect
+      "#<#{self.class.name} columns=#{@headers&.size.inspect} writer=#{@writer.inspect}>"
+    end
+
     private
 
     # @param row [Array, Hash] row given to #<<

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/herringbone/writer.rb

@@ -3,10 +3,10 @@
 module Herringbone
   # Writes Parquet files.
   #
-  #   schema = Herringbone::Schema.define do
-  #     int64 :id, null: false
-  #     string :name
-  #     list :tags, :string
+  #   schema = Herringbone::Schema.define do |s|
+  #     s.int64 :id, null: false
+  #     s.string :name
+  #     s.list :tags, :string
   #   end
   #   File.open("out.parquet", "wb") do |file|
   #     Herringbone::Writer.open(file, schema) do |w|
@@ -24,6 +24,8 @@ module Herringbone
   # Options:
   #   compression:     :snappy (default), :zstd, :gzip, :lz4 (LZ4_RAW), :lz4_hadoop, :brotli, :none
   #                    (:zstd and :brotli need the zstd-ruby / brotli gems)
+  #   compression_level: nil (the codec's default), or a level for :zstd (up to 22), :gzip (0-9)
+  #                    or :brotli (0-11)
   #   row_group_bytes: flush a row group once the buffered values take roughly this much memory
   #                    (default 16MB). This bounds memory use while writing.
   #   row_group_rows:  also flush after this many rows (default: no row limit)
@@ -38,6 +40,9 @@ module Herringbone
   #                    an Array of column paths, or { "path" => true | { ndv:, fpp:, max_bytes: } }.
   #                    Without ndv: the distinct values of each row group are counted. fpp defaults
   #                    to 0.01 and max_bytes to 1MB. Filters are written after each row group.
+  #   encryption:      Parquet modular encryption: { footer_key: "...", columns: { "ssn" => "..." } }.
+  #                    Without columns: every column is encrypted with the footer key. See
+  #                    #initialize for the settings.
   #
   # Statistics and page indexes (ColumnIndex/OffsetIndex) are always written.
   class Writer
@@ -83,6 +88,7 @@ module Herringbone
     # @param schema [Schema] schema of the rows
     # @param options [Hash{Symbol => Object}] see the class description and #initialize
     # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer, nil] :compression_level (nil) level for :zstd, :gzip or :brotli
     # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
     # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
     # @option options [Integer] :page_bytes (1MB) approximate uncompressed data page size
@@ -95,6 +101,8 @@ module Herringbone
     # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
     # @option options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
     #   columns to write split block bloom filters for
+    # @option options [EncryptionConfiguration, Hash{Symbol => Object}, nil] :encryption (nil) modular
+    #   encryption settings, see #initialize
     # @yield [writer] the open writer
     # @yieldparam writer [Writer] writer to append rows to
     # @yieldreturn [Object] returned by open
@@ -118,6 +126,8 @@ module Herringbone
     # @param io [IO, #write] destination; switched to binmode when it supports that
     # @param schema [Schema] schema of the rows
     # @param compression [Symbol, Integer] codec name (see Herringbone.codecs) or Format::Codec id
+    # @param compression_level [Integer, nil] level for :zstd, :gzip or :brotli (see
+    #   Compression::LEVELS), nil for the codec's default
     # @param row_group_bytes [Integer] approximate buffered size that triggers a row group
     # @param row_group_rows [Integer, nil] also flush a row group after this many rows
     # @param page_bytes [Integer] approximate uncompressed data page size
@@ -131,17 +141,35 @@ module Herringbone
     # @param metadata [Hash{#to_s => #to_s}] key/value metadata for the footer
     # @param bloom_filters [Boolean, Array<String>, Hash{String => Boolean, Hash}, nil] see the class
     #   description
-    # @raise [ArgumentError] for an invalid IO, schema or option
+    # @param encryption [EncryptionConfiguration, Hash{Symbol => Object}, false, nil] encrypts the file
+    #   (Parquet modular encryption); a Hash holds the keywords of EncryptionConfiguration.new.
+    #   Keys are 16, 24 or 32-byte Strings; key metadata is stored as given, for readers to find
+    #   the keys by.
+    # @option encryption [String] :footer_key key of the footer, and of columns without their own
+    # @option encryption [String, nil] :footer_key_metadata stored for the footer key
+    # @option encryption [Hash{String => String, Hash, Symbol}, nil] :columns column path or field
+    #   name => key, +{key:, key_metadata:}+ or +:footer+. Columns not listed are not encrypted;
+    #   without this option every column is, with the footer key.
+    # @option encryption [Boolean] :plaintext_footer (false) store the footer in the clear (signed),
+    #   so readers without keys can read the plaintext columns
+    # @option encryption [Symbol] :algorithm (:aes_gcm) or :aes_gcm_ctr, which encrypts pages with
+    #   AES-CTR (faster, but page contents are not authenticated)
+    # @option encryption [String, nil] :aad_prefix identity of the file, which readers can check
+    # @option encryption [Boolean] :store_aad_prefix (true) false leaves it out of the file, so
+    #   readers must supply it
+    # @raise [ArgumentError] for an invalid IO, schema or option, or a compression level the codec
+    #   does not take
     # @raise [MissingCodecError] when the codec's optional gem is not loaded
     # @raise [UnsupportedError] when the codec is not supported
-    def initialize(io, schema, compression: :snappy, row_group_bytes: 16 * 1024 * 1024, row_group_rows: nil,
+    def initialize(io, schema, compression: :snappy, compression_level: nil, row_group_bytes: 16 * 1024 * 1024, row_group_rows: nil,
       page_bytes: 1024 * 1024, page_rows: 20_000, data_page_version: 1, dictionary: true, encodings: {},
-      metadata: {}, bloom_filters: nil)
+      metadata: {}, bloom_filters: nil, encryption: nil)
       raise ArgumentError, "Expected a Herringbone::Schema, got #{schema.class}" unless schema.is_a?(Schema)
       @schema = schema
       @codec = Compression.codec_id(compression)
       # Fail before creating any file if the codec's library is missing
       Compression.ensure_available!(@codec)
+      @compression_level = Compression.check_level!(@codec, compression_level)
       @row_group_bytes = Integer(row_group_bytes)
       @row_group_rows = row_group_rows && Integer(row_group_rows)
       @row_limit = @row_group_rows || ESTIMATE_AFTER_ROWS
@@ -159,14 +187,16 @@ module Herringbone
       @encodings.each { |path, enc| check_encoding!(columns[path], enc) }
       @metadata = metadata
       @bloom_filters = bloom_filter_config(bloom_filters)
-      @pending_bloom_filters = [] # [ColumnMetaData, BloomFilter] for the row group being written
+      @pending_bloom_filters = [] # [ColumnMetaData, BloomFilter, ModuleCrypto] for the row group being written
+      @encryption = encryption ? Encryption::FileEncryptor.new(EncryptionConfiguration.from(encryption), schema) : nil
       @row_groups = []
       @total_rows = 0
       @pos = 0
       @closed = false
+      @aborted = false
       @bytes_per_row = nil
       @io = check_io!(io)
-      write_raw(MAGIC)
+      write_raw(@encryption ? @encryption.magic : MAGIC)
       reset_buffers
     end
 
@@ -222,21 +252,92 @@ module Herringbone
     # @return [void]
     def flush_row_group
       return if @buffered_rows.zero?
+      write_row_group(@buffered_rows)
+    end
+
+    # A column chunk taken as it is from another file, for #write_row_group
+    #
+    # @!attribute chunk
+    #   @return [Format::ColumnChunk] the chunk's footer entry in the source file
+    # @!attribute start
+    #   @return [Integer] source file offset of the chunk's first page
+    # @!attribute bytes
+    #   @return [String] the chunk's pages, headers included
+    # @!attribute column_index
+    #   @return [String, nil] the chunk's encoded ColumnIndex
+    # @!attribute offset_index
+    #   @return [Format::OffsetIndex, nil] the chunk's OffsetIndex, with source file offsets
+    # @!attribute bloom_filter
+    #   @return [String, nil] the chunk's encoded bloom filter, header included
+    CopiedChunk = Struct.new(:chunk, :start, :bytes, :column_index, :offset_index, :bloom_filter)
+
+    # Internal (used by Redaction): writes a row group of +num_rows+ rows from the buffered values,
+    # except for the columns in +copies+, whose chunks are copied byte for byte from another file
+    # with their offsets rebased. Starts a new row group afterwards.
+    #
+    # @param num_rows [Integer] rows in the row group
+    # @param copies [Hash{Integer => CopiedChunk}] column index => chunk to copy instead of encoding
+    #   (only plaintext chunks, into columns this writer does not encrypt)
+    # @param codecs [Hash{Integer => Integer}] column index => codec id for an encoded chunk, instead
+    #   of the +compression:+ option
+    # @param bloom_filters [Hash{Integer => Boolean}] column index => true to give an encoded chunk a
+    #   bloom filter (default settings) when the +bloom_filters:+ option does not ask for one
+    # @param sorting_columns [Array<Format::SortingColumn>, nil] stored on the RowGroup as given
+    # @return [void]
+    # @raise [Error] when the writer is closed
+    def write_row_group(num_rows, copies: {}, codecs: {}, bloom_filters: {}, sorting_columns: nil)
+      raise Error, "Writer is closed" if @closed
+      if @encryption && (encrypted = copies.keys.find { |i| @encryption.encrypted?(i) })
+        raise ArgumentError, "Cannot copy a chunk into the encrypted column #{@schema.columns[encrypted].dotted_path}"
+      end
       start = @pos
-      chunks = @schema.columns.map { |col| write_column_chunk(col, @buffers[col.index]) }
+      chunks = @schema.columns.map do |col|
+        if (copy = copies[col.index])
+          copy_column_chunk(copy)
+        else
+          bloom = @bloom_filters[col.dotted_path]
+          bloom ||= bloom_filter_settings({}) if bloom_filters[col.index] && BloomFilter::TYPES.include?(col.type)
+          write_column_chunk(col, @buffers[col.index], codec: codecs.fetch(col.index, @codec), bloom: bloom)
+        end
+      end
       @row_groups << Format::RowGroup.new(
         columns: chunks,
         total_byte_size: chunks.sum { |c| c.meta_data.total_uncompressed_size },
-        num_rows: @buffered_rows,
+        num_rows: num_rows,
+        sorting_columns: sorting_columns,
         file_offset: start,
         total_compressed_size: @pos - start,
         ordinal: @row_groups.size
       )
       write_bloom_filters
-      @total_rows += @buffered_rows
+      @total_rows += num_rows
       reset_buffers
     end
 
+    # Internal (used by Redaction): whether the column is encrypted in this file
+    #
+    # @param index [Integer] leaf column index
+    # @return [Boolean]
+    def encrypted_column?(index) = @encryption&.encrypted?(index) || false
+
+    # Internal (used by Redaction): shreds one value per row of a top-level field into the buffers,
+    # for #write_row_group. Other fields are left as they are, so the caller decides which columns
+    # are encoded and which are copied.
+    #
+    # @param name [String] top-level field name
+    # @param values [Array] the field's value for each row
+    # @return [void]
+    # @raise [ArgumentError] when the schema has no such field
+    # @raise [EncodeError] when a value cannot be written to the field
+    def buffer_field(name, values)
+      field = @schema.field(name) or raise ArgumentError, "No such field #{name.inspect}"
+      values.each_with_index do |value, i|
+        shred(field, value, 0, 0)
+      rescue EncodeError => e
+        raise EncodeError.new("Row #{@total_rows + i}: #{e.message}", row: @total_rows + i, column: e.column, value: e.value)
+      end
+    end
+
     # Flushes buffered rows, then writes the page indexes and the footer and flushes the IO.
     # Does nothing when already closed or aborted. The IO itself is not closed.
     #
@@ -245,6 +346,7 @@ module Herringbone
       return if @closed
       flush_row_group
       write_page_indexes
+      @encryption&.finish(@row_groups)
       meta = Format::FileMetaData.new(
         version: 2,
         schema: @schema.to_elements,
@@ -254,10 +356,10 @@ module Herringbone
         created_by: "herringbone-ruby #{VERSION}",
         column_orders: @schema.columns.map { Format::ColumnOrder.new(type_order: Format::TypeDefinedOrder.new) }
       )
-      footer = meta.encode
+      footer = @encryption ? @encryption.footer(meta) : meta.encode
       write_raw(footer)
       write_raw([footer.bytesize].pack("V"))
-      write_raw(MAGIC)
+      write_raw(@encryption ? @encryption.magic : MAGIC)
       @io.flush if @io.respond_to?(:flush)
       @closed = true
     end
@@ -267,9 +369,23 @@ module Herringbone
     #
     # @return [void]
     def abort
+      @aborted = true unless @closed
       @closed = true
     end
 
+    # Short summary for the console, without the buffered values
+    #
+    # @return [String] state (open, closed or aborted), rows written including buffered ones,
+    #   row groups flushed and the codec
+    def inspect
+      state = if @aborted then "aborted"
+      elsif @closed then "closed"
+      else "open"
+      end
+      "#<#{self.class.name} #{state} rows_written=#{rows_written} row_groups=#{@row_groups.size} " \
+        "compression=#{Compression::NAMES.fetch(@codec, @codec).inspect}>"
+    end
+
     private
 
     # Pathname responds to #write too (writing a whole file by path), so it is rejected explicitly.
@@ -534,8 +650,11 @@ module Herringbone
     #
     # @param col [Schema::Column] column being written
     # @param buffer [ColumnBuffer] the column's buffered levels and values
+    # @param codec [Integer] codec id to compress the pages with
+    # @param bloom [Hash{Symbol => Numeric, nil}, nil] bloom filter settings (see
+    #   #bloom_filter_settings), nil for no bloom filter
     # @return [Format::ColumnChunk] chunk with its ColumnMetaData, for the row group
-    def write_column_chunk(col, buffer)
+    def write_column_chunk(col, buffer, codec: @codec, bloom: @bloom_filters[col.dotted_path])
       type = col.type
       path = col.dotted_path
       dict_values = nil
@@ -558,6 +677,7 @@ module Herringbone
         @encodings[path] || E::PLAIN
       end
 
+      crypto = @encryption&.chunk(@row_groups.size, col.index)
       chunk_start = @pos
       uncompressed_total = 0
       dictionary_offset = nil
@@ -568,7 +688,7 @@ module Herringbone
           type: Format::PageType::DICTIONARY_PAGE,
           dictionary_page_header: Format::DictionaryPageHeader.new(num_values: dict_values.size, encoding: E::PLAIN)
         )
-        uncompressed_total += write_page(header, plain)
+        uncompressed_total += write_page(header, plain, codec: codec, crypto: crypto)
       end
 
       data_offset = @pos
@@ -601,11 +721,12 @@ module Herringbone
         encoded = encode_values(page_values, value_encoding, type, col.type_length, dict_values&.size)
         rep_bytes = max_rep.positive? ? Encodings::RLE.encode_hybrid(reps, max_rep.bit_length) : "".b
         def_bytes = max_def.positive? ? Encodings::RLE.encode_hybrid(defs, max_def.bit_length) : "".b
+        page_crypto = crypto && [crypto, pages.size]
         uncompressed_total += if @data_page_version == 1
-          write_data_page_v1(n, rep_bytes, def_bytes, encoded, value_encoding)
+          write_data_page_v1(n, rep_bytes, def_bytes, encoded, value_encoding, codec, page_crypto)
         else
           num_rows = max_rep.zero? ? n : reps.count(0)
-          write_data_page_v2(n, n - non_null, num_rows, rep_bytes, def_bytes, encoded, value_encoding)
+          write_data_page_v2(n, n - non_null, num_rows, rep_bytes, def_bytes, encoded, value_encoding, codec, page_crypto)
         end
         pages << PageInfo.new(page_offset, @pos - page_offset, first_row, n - non_null, non_null, range)
         first_row += max_rep.zero? ? n : reps.count(0)
@@ -618,7 +739,7 @@ module Herringbone
         type: type,
         encodings: encodings.uniq,
         path_in_schema: col.path,
-        codec: @codec,
+        codec: codec,
         num_values: buf.defs.size,
         total_uncompressed_size: uncompressed_total,
         total_compressed_size: @pos - chunk_start,
@@ -627,10 +748,10 @@ module Herringbone
         statistics: statistics_for(col, buf.defs, values || indices.uniq.map { |i| dict_values[i] }, order)
       )
       chunk = Format::ColumnChunk.new(file_offset: chunk_start, meta_data: meta)
-      if (bloom = @bloom_filters[path])
-        @pending_bloom_filters << [meta, build_bloom_filter(col, bloom, dict_values, values)]
+      if bloom
+        @pending_bloom_filters << [meta, build_bloom_filter(col, bloom, dict_values, values), crypto]
       end
-      @page_indexes << [chunk, column_index_for(col, pages, order), offset_index_for(pages)]
+      @page_indexes << [chunk, column_index_for(col, pages, order), offset_index_for(pages), crypto]
       chunk
     end
 
@@ -658,6 +779,37 @@ module Herringbone
       end)
     end
 
+    # Writes a chunk copied from another file. The pages, the ColumnIndex and the bloom filter
+    # hold no file offsets and go out as they are; the ColumnMetaData and the OffsetIndex do, so
+    # those are rebased onto where the chunk lands in this file.
+    #
+    # @param copy [CopiedChunk] the chunk to copy
+    # @return [Format::ColumnChunk] chunk with its ColumnMetaData, for the row group
+    def copy_column_chunk(copy)
+      start = @pos
+      shift = start - copy.start
+      meta = Format::ColumnMetaData.decode(copy.chunk.meta_data.encode).first
+      meta.data_page_offset += shift
+      dict = meta.dictionary_page_offset
+      # Some writers store 0 when there is no dictionary page
+      meta.dictionary_page_offset = dict&.positive? ? dict + shift : nil
+      meta.index_page_offset += shift if meta.index_page_offset
+      meta.total_compressed_size = copy.bytes.bytesize
+      meta.bloom_filter_offset = meta.bloom_filter_length = nil
+      write_raw(copy.bytes)
+      chunk = Format::ColumnChunk.new(file_offset: start, meta_data: meta)
+      @pending_bloom_filters << [meta, copy.bloom_filter] if copy.bloom_filter
+      offset_index = copy.offset_index && Format::OffsetIndex.new(
+        page_locations: copy.offset_index.page_locations.map do |loc|
+          Format::PageLocation.new(offset: loc.offset + shift, compressed_page_size: loc.compressed_page_size,
+            first_row_index: loc.first_row_index)
+        end,
+        unencoded_byte_array_data_bytes: copy.offset_index.unencoded_byte_array_data_bytes
+      )
+      @page_indexes << [chunk, copy.column_index, offset_index]
+      chunk
+    end
+
     # nil when the column has no defined sort order, or a page's values have no min/max (all NaN)
     #
     # @param col [Schema::Column] column the pages belong to
@@ -693,19 +845,24 @@ module Herringbone
       end
     end
 
-    # Page indexes go after the last row group: all column indexes, then all offset indexes
+    # Page indexes go after the last row group: all column indexes, then all offset indexes.
+    # A copied chunk brings its ColumnIndex already encoded, and may come without either index.
+    # Those of encrypted columns are encrypted.
     #
     # @return [void]
     def write_page_indexes
-      @page_indexes.each do |chunk, column_index, _|
+      @page_indexes.each do |chunk, column_index, _, crypto|
         next unless column_index
-        bytes = column_index.encode
+        bytes = column_index.is_a?(String) ? column_index : column_index.encode
+        bytes = crypto.encrypt(Encryption::COLUMN_INDEX, bytes) if crypto
         chunk.column_index_offset = @pos
         chunk.column_index_length = bytes.bytesize
         write_raw(bytes)
       end
-      @page_indexes.each do |chunk, _, offset_index|
+      @page_indexes.each do |chunk, _, offset_index, crypto|
+        next unless offset_index
         bytes = offset_index.encode
+        bytes = crypto.encrypt(Encryption::OFFSET_INDEX, bytes) if crypto
         chunk.offset_index_offset = @pos
         chunk.offset_index_length = bytes.bytesize
         write_raw(bytes)
@@ -780,12 +937,19 @@ module Herringbone
       filter
     end
 
-    # Bloom filters go right after the row group's column chunks, in column order
+    # Bloom filters go right after the row group's column chunks, in column order. A copied
+    # chunk brings its filter already encoded. In an encrypted column the header and the bitset
+    # are encrypted separately.
     #
     # @return [void]
     def write_bloom_filters
-      @pending_bloom_filters.each do |meta, filter|
-        bytes = filter.encode
+      @pending_bloom_filters.each do |meta, filter, crypto|
+        bytes = if crypto
+          crypto.encrypt(Encryption::BLOOM_FILTER_HEADER, filter.header.encode) <<
+            crypto.encrypt(Encryption::BLOOM_FILTER_BITSET, filter.bitset)
+        else
+          filter.is_a?(String) ? filter : filter.encode
+        end
         meta.bloom_filter_offset = @pos
         meta.bloom_filter_length = bytes.bytesize
         write_raw(bytes)
@@ -902,17 +1066,40 @@ module Herringbone
 
     # Writes a page, returning the uncompressed size including the header
     #
+    # Compresses with the configured level when +codec+ is the writer's own; chunks a redaction
+    # keeps in their source codec use that codec's default
+    #
+    # @param codec [Integer] codec id
+    # @param data [String] bytes to compress
+    # @return [String] compressed bytes
+    def compress(codec, data)
+      Compression.compress(codec, data, (codec == @codec) ? @compression_level : nil)
+    end
+
+    # In an encrypted column the body and the header are encrypted; the CRC covers the body as
+    # stored, encrypted.
+    #
     # @param header [Format::PageHeader] header; sizes and CRC32 are filled in here
     # @param body [String] uncompressed page body
     # @param compressed [String, nil] bytes to write as the page body, when already prepared (v2 data
     #   pages, whose levels stay uncompressed); +body+ is compressed otherwise
+    # @param codec [Integer] codec id to compress +body+ with
+    # @param crypto [Encryption::ModuleCrypto, nil] encryption of the column's modules
+    # @param ordinal [Integer, nil] data page ordinal within the chunk; nil for a dictionary page
     # @return [Integer]
-    def write_page(header, body, compressed = nil)
-      compressed ||= Compression.compress(@codec, body)
+    def write_page(header, body, compressed = nil, codec: @codec, crypto: nil, ordinal: nil)
+      compressed ||= compress(codec, body)
       header.uncompressed_page_size ||= body.bytesize
+      dictionary = header.type == Format::PageType::DICTIONARY_PAGE
+      if crypto
+        compressed = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE, compressed, ordinal)
+      end
       header.compressed_page_size = compressed.bytesize
       header.crc = Zlib.crc32(compressed).then { |c| (c >= 0x8000_0000) ? c - 0x1_0000_0000 : c }
       encoded = header.encode
+      if crypto
+        encoded = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE_HEADER : Encryption::DATA_PAGE_HEADER, encoded, ordinal)
+      end
       write_raw(encoded)
       write_raw(compressed)
       encoded.bytesize + header.uncompressed_page_size
@@ -925,8 +1112,11 @@ module Herringbone
     # @param def_bytes [String] RLE-encoded definition levels, empty when the column has none
     # @param encoded [String] encoded values
     # @param encoding [Integer] encoding id of the values
+    # @param codec [Integer] codec id to compress the page with
+    # @param page_crypto [Array(Encryption::ModuleCrypto, Integer), nil] encryption of the column's
+    #   modules and the page's ordinal, for an encrypted column
     # @return [Integer] uncompressed size including the header
-    def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding)
+    def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding, codec, page_crypto)
       body = String.new(encoding: Encoding::BINARY)
       body << [rep_bytes.bytesize].pack("V") << rep_bytes unless rep_bytes.empty?
       body << [def_bytes.bytesize].pack("V") << def_bytes unless def_bytes.empty?
@@ -938,7 +1128,7 @@ module Herringbone
           definition_level_encoding: E::RLE, repetition_level_encoding: E::RLE
         )
       )
-      write_page(header, body)
+      write_page(header, body, codec: codec, crypto: page_crypto&.first, ordinal: page_crypto&.last)
     end
 
     # A DATA_PAGE_V2: levels without length prefixes and uncompressed, then the compressed values
@@ -950,9 +1140,12 @@ module Herringbone
     # @param def_bytes [String] RLE-encoded definition levels, empty when the column has none
     # @param encoded [String] encoded values
     # @param encoding [Integer] encoding id of the values
+    # @param codec [Integer] codec id to compress the values with
+    # @param page_crypto [Array(Encryption::ModuleCrypto, Integer), nil] encryption of the column's
+    #   modules and the page's ordinal, for an encrypted column
     # @return [Integer] uncompressed size including the header
-    def write_data_page_v2(n, nulls, rows, rep_bytes, def_bytes, encoded, encoding)
-      compressed = Compression.compress(@codec, encoded)
+    def write_data_page_v2(n, nulls, rows, rep_bytes, def_bytes, encoded, encoding, codec, page_crypto)
+      compressed = compress(codec, encoded)
       header = Format::PageHeader.new(
         type: Format::PageType::DATA_PAGE_V2,
         uncompressed_page_size: rep_bytes.bytesize + def_bytes.bytesize + encoded.bytesize,
@@ -960,10 +1153,11 @@ module Herringbone
           num_values: n, num_nulls: nulls, num_rows: rows, encoding: encoding,
           definition_levels_byte_length: def_bytes.bytesize,
           repetition_levels_byte_length: rep_bytes.bytesize,
-          is_compressed: @codec != Format::Codec::UNCOMPRESSED
+          is_compressed: codec != Format::Codec::UNCOMPRESSED
         )
       )
-      write_page(header, "".b, rep_bytes + def_bytes + compressed)
+      write_page(header, "".b, rep_bytes + def_bytes + compressed, codec: codec, crypto: page_crypto&.first,
+        ordinal: page_crypto&.last)
     end
 
     # Statistics and column index bounds longer than this are truncated, see #truncate_min