herringbone 0.5.0 → 0.6.1

data/lib/herringbone/thrift.rb

@@ -45,6 +45,14 @@ module Herringbone
       double: T_DOUBLE, binary: T_BINARY, string: T_BINARY
     }.freeze
 
+    # Values each declared integer type can hold, whatever width it came in on the wire
+    INT_RANGES = {
+      i16: -2**15...2**15, i32: -2**31...2**31, i64: -2**63...2**63
+    }.freeze
+
+    # Deepest nesting of structs, lists, sets and maps read or skipped, as in Apache Thrift
+    MAX_DEPTH = 64
+
     # Wire type written for a declared field type
     #
     # @param type [Symbol, Array, Class] a scalar type from WIRE_TYPES, a +[:list, elem]+ Array,
@@ -89,6 +97,7 @@ module Herringbone
       def initialize(buf, pos = 0)
         @buf = buf
         @pos = pos
+        @depth = 0
       end
 
       # Reads one raw byte
@@ -104,17 +113,21 @@ module Herringbone
 
       # Reads an unsigned LEB128 varint
       #
-      # @return [Integer] the decoded non-negative value
-      # @raise [Error] when the varint runs past the buffer or is longer than 64 bits allow
+      # @return [Integer] the decoded non-negative value, below 2**64
+      # @raise [Error] when the varint runs past the buffer, is longer than 10 bytes or does not
+      #   fit in 64 bits
       def read_varint
         result = 0
         shift = 0
         while true
           b = read_byte
           result |= (b & 0x7F) << shift
-          return result if b < 0x80
+          if b < 0x80
+            raise Error, "Varint exceeds 64 bits" if result >= 2**64
+            return result
+          end
           shift += 7
-          raise Error, "Varint too long" if shift > 70
+          raise Error, "Varint too long" if shift > 63
         end
       end
 
@@ -141,8 +154,10 @@ module Herringbone
 
       # Reads an 8-byte little-endian double
       #
-      # @return [Float, nil] the value (nil when fewer than 8 bytes are left: this is not checked)
+      # @return [Float] the value
+      # @raise [Error] when fewer than 8 bytes are left
       def read_double
+        raise Error, "Unexpected end of Thrift data at #{@pos}" if @pos + 8 > @buf.bytesize
         v = @buf.byteslice(@pos, 8).unpack1("E")
         @pos += 8
         v
@@ -155,26 +170,28 @@ module Herringbone
       #
       # @param klass [Class] a Thrift::Struct subclass
       # @return [Struct] an instance of +klass+ with the fields that were present set
-      # @raise [Error] on truncated or malformed data
+      # @raise [Error] on truncated or malformed data, or nesting deeper than MAX_DEPTH
       def read_struct(klass)
-        obj = klass.new
-        fields = klass.fields_by_id
-        last_id = 0
-        while true
-          header = read_byte
-          wire = header & 0x0F
-          break if wire == T_STOP
-          delta = header >> 4
-          fid = delta.zero? ? read_zigzag : last_id + delta
-          last_id = fid
-          field = fields[fid]
-          if field && Thrift.compatible?(wire, field.type)
-            obj.instance_variable_set(field.ivar, read_value(wire, field.type))
-          else
-            skip(wire)
+        nested do
+          obj = klass.new
+          fields = klass.fields_by_id
+          last_id = 0
+          while true
+            header = read_byte
+            wire = header & 0x0F
+            break if wire == T_STOP
+            delta = header >> 4
+            fid = delta.zero? ? read_zigzag : last_id + delta
+            last_id = fid
+            field = fields[fid]
+            if field && Thrift.compatible?(wire, field.type)
+              obj.instance_variable_set(field.ivar, read_value(wire, field.type))
+            else
+              skip(wire)
+            end
           end
+          obj
         end
-        obj
       end
 
       # Reads one value of wire type +wire+ as declared +type+
@@ -184,7 +201,8 @@ module Herringbone
       #   an Array or Struct subclass gives the element type or struct to read
       # @return [Object] true/false, an Integer (bytes are signed), a Float, a String, an Array
       #   (or nil, see #read_list) or a Struct
-      # @raise [Error] on an unsupported wire type or truncated data
+      # @raise [Error] on an unsupported wire type, truncated data or an integer out of the range
+      #   of +type+
       def read_value(wire, type)
         case wire
         when T_TRUE then true
@@ -192,7 +210,10 @@ module Herringbone
         when T_BYTE
           b = read_byte
           (b >= 0x80) ? b - 0x100 : b
-        when T_I16, T_I32, T_I64 then read_zigzag
+        when T_I16, T_I32, T_I64
+          n = read_zigzag
+          raise Error, "Integer #{n} out of range for #{type}" unless INT_RANGES.fetch(type).cover?(n)
+          n
         when T_DOUBLE then read_double
         when T_BINARY
           s = read_binary
@@ -210,24 +231,26 @@ module Herringbone
       # @param type [Array] declared list type, +[:list, elem_type]+
       # @return [Array, nil] the elements, or nil (with the list skipped) when the element wire
       #   type does not match +elem_type+
-      # @raise [Error] on truncated or malformed data
+      # @raise [Error] on truncated or malformed data, or nesting deeper than MAX_DEPTH
       def read_list(type)
-        header = read_byte
-        size = header >> 4
-        size = read_varint if size == 15
-        elem_wire = header & 0x0F
-        elem_type = type[1]
-        bool_elems = elem_wire == T_TRUE || elem_wire == T_FALSE
-        if size.positive? && !(bool_elems && elem_type == :bool) && !Thrift.compatible?(elem_wire, elem_type)
-          size.times { bool_elems ? read_byte : skip(elem_wire) }
-          return nil
-        end
-        Array.new(size) do
-          if elem_wire == T_TRUE || elem_wire == T_FALSE
-            # Booleans inside lists are encoded as full bytes
-            read_byte == T_TRUE
-          else
-            read_value(elem_wire, elem_type)
+        nested do
+          header = read_byte
+          size = header >> 4
+          size = read_size if size == 15
+          elem_wire = header & 0x0F
+          elem_type = type[1]
+          bool_elems = elem_wire == T_TRUE || elem_wire == T_FALSE
+          if size.positive? && !(bool_elems && elem_type == :bool) && !Thrift.compatible?(elem_wire, elem_type)
+            size.times { bool_elems ? read_byte : skip(elem_wire) }
+            return nil
+          end
+          Array.new(size) do
+            if elem_wire == T_TRUE || elem_wire == T_FALSE
+              # Booleans inside lists are encoded as full bytes
+              read_byte == T_TRUE
+            else
+              read_value(elem_wire, elem_type)
+            end
           end
         end
       end
@@ -236,40 +259,76 @@ module Herringbone
       #
       # @param wire [Integer] wire type of the value to skip
       # @return [void]
-      # @raise [Error] on an unknown wire type (including T_STOP) or truncated data
+      # @raise [Error] on an unknown wire type (including T_STOP), truncated data or nesting deeper
+      #   than MAX_DEPTH
       def skip(wire)
         case wire
         when T_TRUE, T_FALSE then nil
         when T_BYTE then read_byte
         when T_I16, T_I32, T_I64 then read_varint
-        when T_DOUBLE then @pos += 8
+        when T_DOUBLE then read_double
         when T_BINARY then read_binary
         when T_LIST, T_SET
-          header = read_byte
-          size = header >> 4
-          size = read_varint if size == 15
-          elem = header & 0x0F
-          size.times { (elem == T_TRUE || elem == T_FALSE) ? read_byte : skip(elem) }
+          nested do
+            header = read_byte
+            size = header >> 4
+            size = read_size if size == 15
+            elem = header & 0x0F
+            size.times { (elem == T_TRUE || elem == T_FALSE) ? read_byte : skip(elem) }
+          end
         when T_MAP
-          size = read_varint
-          unless size.zero?
-            kv = read_byte
-            size.times do
-              [kv >> 4, kv & 0x0F].each { |w| (w == T_TRUE || w == T_FALSE) ? read_byte : skip(w) }
+          nested do
+            size = read_size
+            unless size.zero?
+              kv = read_byte
+              size.times do
+                [kv >> 4, kv & 0x0F].each { |w| (w == T_TRUE || w == T_FALSE) ? read_byte : skip(w) }
+              end
             end
           end
         when T_STRUCT
-          while true
-            header = read_byte
-            w = header & 0x0F
-            break if w == T_STOP
-            read_zigzag if (header >> 4).zero?
-            skip(w)
+          nested do
+            while true
+              header = read_byte
+              w = header & 0x0F
+              break if w == T_STOP
+              read_zigzag if (header >> 4).zero?
+              skip(w)
+            end
           end
         else
           raise Error, "Cannot skip wire type #{wire}"
         end
       end
+
+      private
+
+      # Reads the element count of a list, set or map
+      #
+      # Every element takes at least one byte, so a count above the bytes left is corrupt. Checking
+      # that up front keeps a forged count from preallocating a huge Array or skipping for ever.
+      #
+      # @return [Integer] the count, at most the number of bytes left in the buffer
+      # @raise [Error] when the count exceeds the bytes left
+      def read_size
+        size = read_varint
+        left = @buf.bytesize - @pos
+        raise Error, "Collection of #{size} elements exceeds the #{left} bytes left" if size > left
+        size
+      end
+
+      # Runs the block one nesting level deeper
+      #
+      # @yieldreturn [Object] the value read inside the nested container
+      # @return [Object] the block's value
+      # @raise [Error] when the nesting gets deeper than MAX_DEPTH
+      def nested
+        @depth += 1
+        raise Error, "Thrift data nested deeper than #{MAX_DEPTH} levels" if @depth > MAX_DEPTH
+        yield
+      ensure
+        @depth -= 1
+      end
     end
 
     # Encodes compact protocol data by appending to a binary String

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.5.0"
+  VERSION = "0.6.1"
 end

data/lib/herringbone/writer.rb

@@ -24,6 +24,8 @@ module Herringbone
   # Options:
   #   compression:     :snappy (default), :zstd, :gzip, :lz4 (LZ4_RAW), :lz4_hadoop, :brotli, :none
   #                    (:zstd and :brotli need the zstd-ruby / brotli gems)
+  #   compression_level: nil (the codec's default), or a level for :zstd (up to 22), :gzip (0-9)
+  #                    or :brotli (0-11)
   #   row_group_bytes: flush a row group once the buffered values take roughly this much memory
   #                    (default 16MB). This bounds memory use while writing.
   #   row_group_rows:  also flush after this many rows (default: no row limit)
@@ -38,6 +40,9 @@ module Herringbone
   #                    an Array of column paths, or { "path" => true | { ndv:, fpp:, max_bytes: } }.
   #                    Without ndv: the distinct values of each row group are counted. fpp defaults
   #                    to 0.01 and max_bytes to 1MB. Filters are written after each row group.
+  #   encryption:      Parquet modular encryption: { footer_key: "...", columns: { "ssn" => "..." } }.
+  #                    Without columns: every column is encrypted with the footer key. See
+  #                    #initialize for the settings.
   #
   # Statistics and page indexes (ColumnIndex/OffsetIndex) are always written.
   class Writer
@@ -83,6 +88,7 @@ module Herringbone
     # @param schema [Schema] schema of the rows
     # @param options [Hash{Symbol => Object}] see the class description and #initialize
     # @option options [Symbol] :compression (:snappy) codec, see Herringbone.codecs
+    # @option options [Integer, nil] :compression_level (nil) level for :zstd, :gzip or :brotli
     # @option options [Integer] :row_group_bytes (16MB) approximate buffered size that triggers a row group
     # @option options [Integer, nil] :row_group_rows (nil) also flush a row group after this many rows
     # @option options [Integer] :page_bytes (1MB) approximate uncompressed data page size
@@ -95,6 +101,8 @@ module Herringbone
     # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
     # @option options [Boolean, Array<String>, Hash{String => Boolean, Hash}] :bloom_filters (nil)
     #   columns to write split block bloom filters for
+    # @option options [EncryptionConfiguration, Hash{Symbol => Object}, nil] :encryption (nil) modular
+    #   encryption settings, see #initialize
     # @yield [writer] the open writer
     # @yieldparam writer [Writer] writer to append rows to
     # @yieldreturn [Object] returned by open
@@ -118,6 +126,8 @@ module Herringbone
     # @param io [IO, #write] destination; switched to binmode when it supports that
     # @param schema [Schema] schema of the rows
     # @param compression [Symbol, Integer] codec name (see Herringbone.codecs) or Format::Codec id
+    # @param compression_level [Integer, nil] level for :zstd, :gzip or :brotli (see
+    #   Compression::LEVELS), nil for the codec's default
     # @param row_group_bytes [Integer] approximate buffered size that triggers a row group
     # @param row_group_rows [Integer, nil] also flush a row group after this many rows
     # @param page_bytes [Integer] approximate uncompressed data page size
@@ -131,17 +141,35 @@ module Herringbone
     # @param metadata [Hash{#to_s => #to_s}] key/value metadata for the footer
     # @param bloom_filters [Boolean, Array<String>, Hash{String => Boolean, Hash}, nil] see the class
     #   description
-    # @raise [ArgumentError] for an invalid IO, schema or option
+    # @param encryption [EncryptionConfiguration, Hash{Symbol => Object}, false, nil] encrypts the file
+    #   (Parquet modular encryption); a Hash holds the keywords of EncryptionConfiguration.new.
+    #   Keys are 16, 24 or 32-byte Strings; key metadata is stored as given, for readers to find
+    #   the keys by.
+    # @option encryption [String] :footer_key key of the footer, and of columns without their own
+    # @option encryption [String, nil] :footer_key_metadata stored for the footer key
+    # @option encryption [Hash{String => String, Hash, Symbol}, nil] :columns column path or field
+    #   name => key, +{key:, key_metadata:}+ or +:footer+. Columns not listed are not encrypted;
+    #   without this option every column is, with the footer key.
+    # @option encryption [Boolean] :plaintext_footer (false) store the footer in the clear (signed),
+    #   so readers without keys can read the plaintext columns
+    # @option encryption [Symbol] :algorithm (:aes_gcm) or :aes_gcm_ctr, which encrypts pages with
+    #   AES-CTR (faster, but page contents are not authenticated)
+    # @option encryption [String, nil] :aad_prefix identity of the file, which readers can check
+    # @option encryption [Boolean] :store_aad_prefix (true) false leaves it out of the file, so
+    #   readers must supply it
+    # @raise [ArgumentError] for an invalid IO, schema or option, or a compression level the codec
+    #   does not take
     # @raise [MissingCodecError] when the codec's optional gem is not loaded
     # @raise [UnsupportedError] when the codec is not supported
-    def initialize(io, schema, compression: :snappy, row_group_bytes: 16 * 1024 * 1024, row_group_rows: nil,
+    def initialize(io, schema, compression: :snappy, compression_level: nil, row_group_bytes: 16 * 1024 * 1024, row_group_rows: nil,
       page_bytes: 1024 * 1024, page_rows: 20_000, data_page_version: 1, dictionary: true, encodings: {},
-      metadata: {}, bloom_filters: nil)
+      metadata: {}, bloom_filters: nil, encryption: nil)
       raise ArgumentError, "Expected a Herringbone::Schema, got #{schema.class}" unless schema.is_a?(Schema)
       @schema = schema
       @codec = Compression.codec_id(compression)
       # Fail before creating any file if the codec's library is missing
       Compression.ensure_available!(@codec)
+      @compression_level = Compression.check_level!(@codec, compression_level)
       @row_group_bytes = Integer(row_group_bytes)
       @row_group_rows = row_group_rows && Integer(row_group_rows)
       @row_limit = @row_group_rows || ESTIMATE_AFTER_ROWS
@@ -159,7 +187,8 @@ module Herringbone
       @encodings.each { |path, enc| check_encoding!(columns[path], enc) }
       @metadata = metadata
       @bloom_filters = bloom_filter_config(bloom_filters)
-      @pending_bloom_filters = [] # [ColumnMetaData, BloomFilter] for the row group being written
+      @pending_bloom_filters = [] # [ColumnMetaData, BloomFilter, ModuleCrypto] for the row group being written
+      @encryption = encryption ? Encryption::FileEncryptor.new(EncryptionConfiguration.from(encryption), schema) : nil
       @row_groups = []
       @total_rows = 0
       @pos = 0
@@ -167,7 +196,7 @@ module Herringbone
       @aborted = false
       @bytes_per_row = nil
       @io = check_io!(io)
-      write_raw(MAGIC)
+      write_raw(@encryption ? @encryption.magic : MAGIC)
       reset_buffers
     end
 
@@ -248,6 +277,7 @@ module Herringbone
     #
     # @param num_rows [Integer] rows in the row group
     # @param copies [Hash{Integer => CopiedChunk}] column index => chunk to copy instead of encoding
+    #   (only plaintext chunks, into columns this writer does not encrypt)
     # @param codecs [Hash{Integer => Integer}] column index => codec id for an encoded chunk, instead
     #   of the +compression:+ option
     # @param bloom_filters [Hash{Integer => Boolean}] column index => true to give an encoded chunk a
@@ -257,6 +287,9 @@ module Herringbone
     # @raise [Error] when the writer is closed
     def write_row_group(num_rows, copies: {}, codecs: {}, bloom_filters: {}, sorting_columns: nil)
       raise Error, "Writer is closed" if @closed
+      if @encryption && (encrypted = copies.keys.find { |i| @encryption.encrypted?(i) })
+        raise ArgumentError, "Cannot copy a chunk into the encrypted column #{@schema.columns[encrypted].dotted_path}"
+      end
       start = @pos
       chunks = @schema.columns.map do |col|
         if (copy = copies[col.index])
@@ -281,6 +314,12 @@ module Herringbone
       reset_buffers
     end
 
+    # Internal (used by Redaction): whether the column is encrypted in this file
+    #
+    # @param index [Integer] leaf column index
+    # @return [Boolean]
+    def encrypted_column?(index) = @encryption&.encrypted?(index) || false
+
     # Internal (used by Redaction): shreds one value per row of a top-level field into the buffers,
     # for #write_row_group. Other fields are left as they are, so the caller decides which columns
     # are encoded and which are copied.
@@ -307,6 +346,7 @@ module Herringbone
       return if @closed
       flush_row_group
       write_page_indexes
+      @encryption&.finish(@row_groups)
       meta = Format::FileMetaData.new(
         version: 2,
         schema: @schema.to_elements,
@@ -316,10 +356,10 @@ module Herringbone
         created_by: "herringbone-ruby #{VERSION}",
         column_orders: @schema.columns.map { Format::ColumnOrder.new(type_order: Format::TypeDefinedOrder.new) }
       )
-      footer = meta.encode
+      footer = @encryption ? @encryption.footer(meta) : meta.encode
       write_raw(footer)
       write_raw([footer.bytesize].pack("V"))
-      write_raw(MAGIC)
+      write_raw(@encryption ? @encryption.magic : MAGIC)
       @io.flush if @io.respond_to?(:flush)
       @closed = true
     end
@@ -637,6 +677,7 @@ module Herringbone
         @encodings[path] || E::PLAIN
       end
 
+      crypto = @encryption&.chunk(@row_groups.size, col.index)
       chunk_start = @pos
       uncompressed_total = 0
       dictionary_offset = nil
@@ -647,7 +688,7 @@ module Herringbone
           type: Format::PageType::DICTIONARY_PAGE,
           dictionary_page_header: Format::DictionaryPageHeader.new(num_values: dict_values.size, encoding: E::PLAIN)
         )
-        uncompressed_total += write_page(header, plain, codec: codec)
+        uncompressed_total += write_page(header, plain, codec: codec, crypto: crypto)
       end
 
       data_offset = @pos
@@ -680,11 +721,12 @@ module Herringbone
         encoded = encode_values(page_values, value_encoding, type, col.type_length, dict_values&.size)
         rep_bytes = max_rep.positive? ? Encodings::RLE.encode_hybrid(reps, max_rep.bit_length) : "".b
         def_bytes = max_def.positive? ? Encodings::RLE.encode_hybrid(defs, max_def.bit_length) : "".b
+        page_crypto = crypto && [crypto, pages.size]
         uncompressed_total += if @data_page_version == 1
-          write_data_page_v1(n, rep_bytes, def_bytes, encoded, value_encoding, codec)
+          write_data_page_v1(n, rep_bytes, def_bytes, encoded, value_encoding, codec, page_crypto)
         else
           num_rows = max_rep.zero? ? n : reps.count(0)
-          write_data_page_v2(n, n - non_null, num_rows, rep_bytes, def_bytes, encoded, value_encoding, codec)
+          write_data_page_v2(n, n - non_null, num_rows, rep_bytes, def_bytes, encoded, value_encoding, codec, page_crypto)
         end
         pages << PageInfo.new(page_offset, @pos - page_offset, first_row, n - non_null, non_null, range)
         first_row += max_rep.zero? ? n : reps.count(0)
@@ -707,9 +749,9 @@ module Herringbone
       )
       chunk = Format::ColumnChunk.new(file_offset: chunk_start, meta_data: meta)
       if bloom
-        @pending_bloom_filters << [meta, build_bloom_filter(col, bloom, dict_values, values)]
+        @pending_bloom_filters << [meta, build_bloom_filter(col, bloom, dict_values, values), crypto]
       end
-      @page_indexes << [chunk, column_index_for(col, pages, order), offset_index_for(pages)]
+      @page_indexes << [chunk, column_index_for(col, pages, order), offset_index_for(pages), crypto]
       chunk
     end
 
@@ -805,19 +847,22 @@ module Herringbone
 
     # Page indexes go after the last row group: all column indexes, then all offset indexes.
     # A copied chunk brings its ColumnIndex already encoded, and may come without either index.
+    # Those of encrypted columns are encrypted.
     #
     # @return [void]
     def write_page_indexes
-      @page_indexes.each do |chunk, column_index, _|
+      @page_indexes.each do |chunk, column_index, _, crypto|
         next unless column_index
         bytes = column_index.is_a?(String) ? column_index : column_index.encode
+        bytes = crypto.encrypt(Encryption::COLUMN_INDEX, bytes) if crypto
         chunk.column_index_offset = @pos
         chunk.column_index_length = bytes.bytesize
         write_raw(bytes)
       end
-      @page_indexes.each do |chunk, _, offset_index|
+      @page_indexes.each do |chunk, _, offset_index, crypto|
         next unless offset_index
         bytes = offset_index.encode
+        bytes = crypto.encrypt(Encryption::OFFSET_INDEX, bytes) if crypto
         chunk.offset_index_offset = @pos
         chunk.offset_index_length = bytes.bytesize
         write_raw(bytes)
@@ -893,12 +938,18 @@ module Herringbone
     end
 
     # Bloom filters go right after the row group's column chunks, in column order. A copied
-    # chunk brings its filter already encoded.
+    # chunk brings its filter already encoded. In an encrypted column the header and the bitset
+    # are encrypted separately.
     #
     # @return [void]
     def write_bloom_filters
-      @pending_bloom_filters.each do |meta, filter|
-        bytes = filter.is_a?(String) ? filter : filter.encode
+      @pending_bloom_filters.each do |meta, filter, crypto|
+        bytes = if crypto
+          crypto.encrypt(Encryption::BLOOM_FILTER_HEADER, filter.header.encode) <<
+            crypto.encrypt(Encryption::BLOOM_FILTER_BITSET, filter.bitset)
+        else
+          filter.is_a?(String) ? filter : filter.encode
+        end
         meta.bloom_filter_offset = @pos
         meta.bloom_filter_length = bytes.bytesize
         write_raw(bytes)
@@ -1015,18 +1066,40 @@ module Herringbone
 
     # Writes a page, returning the uncompressed size including the header
     #
+    # Compresses with the configured level when +codec+ is the writer's own; chunks a redaction
+    # keeps in their source codec use that codec's default
+    #
+    # @param codec [Integer] codec id
+    # @param data [String] bytes to compress
+    # @return [String] compressed bytes
+    def compress(codec, data)
+      Compression.compress(codec, data, (codec == @codec) ? @compression_level : nil)
+    end
+
+    # In an encrypted column the body and the header are encrypted; the CRC covers the body as
+    # stored, encrypted.
+    #
     # @param header [Format::PageHeader] header; sizes and CRC32 are filled in here
     # @param body [String] uncompressed page body
     # @param compressed [String, nil] bytes to write as the page body, when already prepared (v2 data
     #   pages, whose levels stay uncompressed); +body+ is compressed otherwise
     # @param codec [Integer] codec id to compress +body+ with
+    # @param crypto [Encryption::ModuleCrypto, nil] encryption of the column's modules
+    # @param ordinal [Integer, nil] data page ordinal within the chunk; nil for a dictionary page
     # @return [Integer]
-    def write_page(header, body, compressed = nil, codec: @codec)
-      compressed ||= Compression.compress(codec, body)
+    def write_page(header, body, compressed = nil, codec: @codec, crypto: nil, ordinal: nil)
+      compressed ||= compress(codec, body)
       header.uncompressed_page_size ||= body.bytesize
+      dictionary = header.type == Format::PageType::DICTIONARY_PAGE
+      if crypto
+        compressed = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE, compressed, ordinal)
+      end
       header.compressed_page_size = compressed.bytesize
       header.crc = Zlib.crc32(compressed).then { |c| (c >= 0x8000_0000) ? c - 0x1_0000_0000 : c }
       encoded = header.encode
+      if crypto
+        encoded = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE_HEADER : Encryption::DATA_PAGE_HEADER, encoded, ordinal)
+      end
       write_raw(encoded)
       write_raw(compressed)
       encoded.bytesize + header.uncompressed_page_size
@@ -1040,8 +1113,10 @@ module Herringbone
     # @param encoded [String] encoded values
     # @param encoding [Integer] encoding id of the values
     # @param codec [Integer] codec id to compress the page with
+    # @param page_crypto [Array(Encryption::ModuleCrypto, Integer), nil] encryption of the column's
+    #   modules and the page's ordinal, for an encrypted column
     # @return [Integer] uncompressed size including the header
-    def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding, codec)
+    def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding, codec, page_crypto)
       body = String.new(encoding: Encoding::BINARY)
       body << [rep_bytes.bytesize].pack("V") << rep_bytes unless rep_bytes.empty?
       body << [def_bytes.bytesize].pack("V") << def_bytes unless def_bytes.empty?
@@ -1053,7 +1128,7 @@ module Herringbone
           definition_level_encoding: E::RLE, repetition_level_encoding: E::RLE
         )
       )
-      write_page(header, body, codec: codec)
+      write_page(header, body, codec: codec, crypto: page_crypto&.first, ordinal: page_crypto&.last)
     end
 
     # A DATA_PAGE_V2: levels without length prefixes and uncompressed, then the compressed values
@@ -1066,9 +1141,11 @@ module Herringbone
     # @param encoded [String] encoded values
     # @param encoding [Integer] encoding id of the values
     # @param codec [Integer] codec id to compress the values with
+    # @param page_crypto [Array(Encryption::ModuleCrypto, Integer), nil] encryption of the column's
+    #   modules and the page's ordinal, for an encrypted column
     # @return [Integer] uncompressed size including the header
-    def write_data_page_v2(n, nulls, rows, rep_bytes, def_bytes, encoded, encoding, codec)
-      compressed = Compression.compress(codec, encoded)
+    def write_data_page_v2(n, nulls, rows, rep_bytes, def_bytes, encoded, encoding, codec, page_crypto)
+      compressed = compress(codec, encoded)
       header = Format::PageHeader.new(
         type: Format::PageType::DATA_PAGE_V2,
         uncompressed_page_size: rep_bytes.bytesize + def_bytes.bytesize + encoded.bytesize,
@@ -1079,7 +1156,8 @@ module Herringbone
           is_compressed: codec != Format::Codec::UNCOMPRESSED
         )
       )
-      write_page(header, "".b, rep_bytes + def_bytes + compressed, codec: codec)
+      write_page(header, "".b, rep_bytes + def_bytes + compressed, codec: codec, crypto: page_crypto&.first,
+        ordinal: page_crypto&.last)
     end
 
     # Statistics and column index bounds longer than this are truncated, see #truncate_min