herringbone 0.6.0 → 0.7.0

data/lib/herringbone/thrift.rb

@@ -4,6 +4,8 @@ module Herringbone
   # Minimal Thrift Compact Protocol implementation, just enough for Parquet metadata.
   # Structs are described declaratively (see Herringbone::Thrift::Struct) so that
   # both the reader and the writer are driven by the same field tables.
+  #
+  # @api private
   module Thrift
     # Raised on malformed or truncated Thrift data, and on values the writer cannot encode
     class Error < StandardError; end
@@ -45,6 +47,14 @@ module Herringbone
       double: T_DOUBLE, binary: T_BINARY, string: T_BINARY
     }.freeze
 
+    # Values each declared integer type can hold, whatever width it came in on the wire
+    INT_RANGES = {
+      byte: -2**7...2**7, i16: -2**15...2**15, i32: -2**31...2**31, i64: -2**63...2**63
+    }.freeze
+
+    # Deepest nesting of structs, lists, sets and maps read or skipped, as in Apache Thrift
+    MAX_DEPTH = 64
+
     # Wire type written for a declared field type
     #
     # @param type [Symbol, Array, Class] a scalar type from WIRE_TYPES, a +[:list, elem]+ Array,
@@ -86,9 +96,12 @@ module Herringbone
 
       # @param buf [String] the encoded data (binary)
       # @param pos [Integer] byte offset to start reading at
+      # @raise [Error] when +pos+ is negative
       def initialize(buf, pos = 0)
+        raise Error, "Negative offset #{pos}" if pos.negative?
         @buf = buf
         @pos = pos
+        @depth = 0
       end
 
       # Reads one raw byte
@@ -104,17 +117,21 @@ module Herringbone
 
       # Reads an unsigned LEB128 varint
       #
-      # @return [Integer] the decoded non-negative value
-      # @raise [Error] when the varint runs past the buffer or is longer than 64 bits allow
+      # @return [Integer] the decoded non-negative value, below 2**64
+      # @raise [Error] when the varint runs past the buffer, is longer than 10 bytes or does not
+      #   fit in 64 bits
       def read_varint
         result = 0
         shift = 0
         while true
           b = read_byte
           result |= (b & 0x7F) << shift
-          return result if b < 0x80
+          if b < 0x80
+            raise Error, "Varint exceeds 64 bits" if result >= 2**64
+            return result
+          end
           shift += 7
-          raise Error, "Varint too long" if shift > 70
+          raise Error, "Varint too long" if shift > 63
         end
       end
 
@@ -141,8 +158,10 @@ module Herringbone
 
       # Reads an 8-byte little-endian double
       #
-      # @return [Float, nil] the value (nil when fewer than 8 bytes are left: this is not checked)
+      # @return [Float] the value
+      # @raise [Error] when fewer than 8 bytes are left
       def read_double
+        raise Error, "Unexpected end of Thrift data at #{@pos}" if @pos + 8 > @buf.bytesize
         v = @buf.byteslice(@pos, 8).unpack1("E")
         @pos += 8
         v
@@ -155,36 +174,39 @@ module Herringbone
       #
       # @param klass [Class] a Thrift::Struct subclass
       # @return [Struct] an instance of +klass+ with the fields that were present set
-      # @raise [Error] on truncated or malformed data
+      # @raise [Error] on truncated or malformed data, or nesting deeper than MAX_DEPTH
       def read_struct(klass)
-        obj = klass.new
-        fields = klass.fields_by_id
-        last_id = 0
-        while true
-          header = read_byte
-          wire = header & 0x0F
-          break if wire == T_STOP
-          delta = header >> 4
-          fid = delta.zero? ? read_zigzag : last_id + delta
-          last_id = fid
-          field = fields[fid]
-          if field && Thrift.compatible?(wire, field.type)
-            obj.instance_variable_set(field.ivar, read_value(wire, field.type))
-          else
-            skip(wire)
+        nested do
+          obj = klass.new
+          fields = klass.fields_by_id
+          last_id = 0
+          while true
+            header = read_byte
+            wire = header & 0x0F
+            break if wire == T_STOP
+            delta = header >> 4
+            fid = delta.zero? ? read_zigzag : last_id + delta
+            last_id = fid
+            field = fields[fid]
+            if field && Thrift.compatible?(wire, field.type)
+              obj.instance_variable_set(field.ivar, read_value(wire, field.type))
+            else
+              skip(wire)
+            end
           end
+          obj
         end
-        obj
       end
 
       # Reads one value of wire type +wire+ as declared +type+
       #
       # @param wire [Integer] wire type from the field or list header
-      # @param type [Symbol, Array, Class] declared type; +:string+ makes binary values UTF-8,
-      #   an Array or Struct subclass gives the element type or struct to read
+      # @param type [Symbol, Array, Class] declared type; +:string+ makes binary values UTF-8 and
+      #   +:binary+ makes them BINARY, an Array or Struct subclass gives the element type or struct to read
       # @return [Object] true/false, an Integer (bytes are signed), a Float, a String, an Array
       #   (or nil, see #read_list) or a Struct
-      # @raise [Error] on an unsupported wire type or truncated data
+      # @raise [Error] on an unsupported wire type, truncated data or an integer out of the range
+      #   of +type+
       def read_value(wire, type)
         case wire
         when T_TRUE then true
@@ -192,11 +214,14 @@ module Herringbone
         when T_BYTE
           b = read_byte
           (b >= 0x80) ? b - 0x100 : b
-        when T_I16, T_I32, T_I64 then read_zigzag
+        when T_I16, T_I32, T_I64
+          n = read_zigzag
+          raise Error, "Integer #{n} out of range for #{type}" unless INT_RANGES.fetch(type).cover?(n)
+          n
         when T_DOUBLE then read_double
         when T_BINARY
           s = read_binary
-          (type == :string) ? s.force_encoding(Encoding::UTF_8) : s
+          s.force_encoding((type == :string) ? Encoding::UTF_8 : Encoding::BINARY)
         when T_LIST, T_SET then read_list(type)
         when T_STRUCT then read_struct(type)
         else
@@ -210,24 +235,26 @@ module Herringbone
       # @param type [Array] declared list type, +[:list, elem_type]+
       # @return [Array, nil] the elements, or nil (with the list skipped) when the element wire
       #   type does not match +elem_type+
-      # @raise [Error] on truncated or malformed data
+      # @raise [Error] on truncated or malformed data, or nesting deeper than MAX_DEPTH
       def read_list(type)
-        header = read_byte
-        size = header >> 4
-        size = read_varint if size == 15
-        elem_wire = header & 0x0F
-        elem_type = type[1]
-        bool_elems = elem_wire == T_TRUE || elem_wire == T_FALSE
-        if size.positive? && !(bool_elems && elem_type == :bool) && !Thrift.compatible?(elem_wire, elem_type)
-          size.times { bool_elems ? read_byte : skip(elem_wire) }
-          return nil
-        end
-        Array.new(size) do
-          if elem_wire == T_TRUE || elem_wire == T_FALSE
-            # Booleans inside lists are encoded as full bytes
-            read_byte == T_TRUE
-          else
-            read_value(elem_wire, elem_type)
+        nested do
+          header = read_byte
+          size = header >> 4
+          size = read_size if size == 15
+          elem_wire = header & 0x0F
+          elem_type = type[1]
+          bool_elems = elem_wire == T_TRUE || elem_wire == T_FALSE
+          if size.positive? && !(bool_elems && elem_type == :bool) && !Thrift.compatible?(elem_wire, elem_type)
+            size.times { bool_elems ? read_byte : skip(elem_wire) }
+            return nil
+          end
+          Array.new(size) do
+            if elem_wire == T_TRUE || elem_wire == T_FALSE
+              # Booleans inside lists are encoded as full bytes
+              read_byte == T_TRUE
+            else
+              read_value(elem_wire, elem_type)
+            end
           end
         end
       end
@@ -236,40 +263,76 @@ module Herringbone
       #
       # @param wire [Integer] wire type of the value to skip
       # @return [void]
-      # @raise [Error] on an unknown wire type (including T_STOP) or truncated data
+      # @raise [Error] on an unknown wire type (including T_STOP), truncated data or nesting deeper
+      #   than MAX_DEPTH
       def skip(wire)
         case wire
         when T_TRUE, T_FALSE then nil
         when T_BYTE then read_byte
         when T_I16, T_I32, T_I64 then read_varint
-        when T_DOUBLE then @pos += 8
+        when T_DOUBLE then read_double
         when T_BINARY then read_binary
         when T_LIST, T_SET
-          header = read_byte
-          size = header >> 4
-          size = read_varint if size == 15
-          elem = header & 0x0F
-          size.times { (elem == T_TRUE || elem == T_FALSE) ? read_byte : skip(elem) }
+          nested do
+            header = read_byte
+            size = header >> 4
+            size = read_size if size == 15
+            elem = header & 0x0F
+            size.times { (elem == T_TRUE || elem == T_FALSE) ? read_byte : skip(elem) }
+          end
         when T_MAP
-          size = read_varint
-          unless size.zero?
-            kv = read_byte
-            size.times do
-              [kv >> 4, kv & 0x0F].each { |w| (w == T_TRUE || w == T_FALSE) ? read_byte : skip(w) }
+          nested do
+            size = read_size
+            unless size.zero?
+              kv = read_byte
+              size.times do
+                [kv >> 4, kv & 0x0F].each { |w| (w == T_TRUE || w == T_FALSE) ? read_byte : skip(w) }
+              end
             end
           end
         when T_STRUCT
-          while true
-            header = read_byte
-            w = header & 0x0F
-            break if w == T_STOP
-            read_zigzag if (header >> 4).zero?
-            skip(w)
+          nested do
+            while true
+              header = read_byte
+              w = header & 0x0F
+              break if w == T_STOP
+              read_zigzag if (header >> 4).zero?
+              skip(w)
+            end
           end
         else
           raise Error, "Cannot skip wire type #{wire}"
         end
       end
+
+      private
+
+      # Reads the element count of a list, set or map
+      #
+      # Every element takes at least one byte, so a count above the bytes left is corrupt. Checking
+      # that up front keeps a forged count from preallocating a huge Array or skipping for ever.
+      #
+      # @return [Integer] the count, at most the number of bytes left in the buffer
+      # @raise [Error] when the count exceeds the bytes left
+      def read_size
+        size = read_varint
+        left = @buf.bytesize - @pos
+        raise Error, "Collection of #{size} elements exceeds the #{left} bytes left" if size > left
+        size
+      end
+
+      # Runs the block one nesting level deeper
+      #
+      # @yieldreturn [Object] the value read inside the nested container
+      # @return [Object] the block's value
+      # @raise [Error] when the nesting gets deeper than MAX_DEPTH
+      def nested
+        @depth += 1
+        raise Error, "Thrift data nested deeper than #{MAX_DEPTH} levels" if @depth > MAX_DEPTH
+        yield
+      ensure
+        @depth -= 1
+      end
     end
 
     # Encodes compact protocol data by appending to a binary String
@@ -348,12 +411,15 @@ module Herringbone
       # @param type [Symbol, Array, Class] declared type (see WIRE_TYPES)
       # @param value [Object] Integer, Float, String, Array or Struct matching +type+
       # @return [void]
+      # @raise [Error] when an integer type gets a value that is not an Integer in its range, or a
+      #   +:string+ gets a value that is not valid UTF-8 and cannot be converted to it
       def write_value(type, value)
         case type
-        when :byte then @buf << (value & 0xFF)
-        when :i16, :i32, :i64 then write_zigzag(value)
+        when :byte then @buf << (checked_int(type, value) & 0xFF)
+        when :i16, :i32, :i64 then write_zigzag(checked_int(type, value))
         when :double then @buf << [value].pack("E")
-        when :binary, :string then write_binary(value)
+        when :binary then write_binary(value)
+        when :string then write_binary(utf8(value))
         when Array then write_list(type[1], value)
         else write_struct(value)
         end
@@ -381,6 +447,32 @@ module Herringbone
           end
         end
       end
+
+      private
+
+      # @param type [Symbol] declared integer type, a key of INT_RANGES
+      # @param value [Object] value to be written as +type+
+      # @return [Integer] +value+
+      # @raise [Error] when +value+ is not an Integer that +type+ can hold
+      def checked_int(type, value)
+        raise Error, "#{value.inspect} is not a valid #{type}" unless value.is_a?(Integer) && INT_RANGES.fetch(type).cover?(value)
+        value
+      end
+
+      # Parquet strings are UTF-8. Binary Strings are taken to hold UTF-8 bytes, Strings in other
+      # encodings are converted.
+      #
+      # @param s [String] value of a +:string+ field
+      # @return [String] +s+, or a UTF-8 copy of it
+      # @raise [Error] when +s+ is not valid UTF-8 and cannot be converted to it
+      def utf8(s)
+        return s if s.ascii_only? || (s.encoding == Encoding::UTF_8 && s.valid_encoding?)
+        u = (s.encoding == Encoding::BINARY) ? s.dup.force_encoding(Encoding::UTF_8) : s.encode(Encoding::UTF_8)
+        raise Error, "String in #{s.encoding} is not valid UTF-8" unless u.valid_encoding?
+        u
+      rescue EncodingError => e
+        raise Error, "String in #{s.encoding} cannot be converted to UTF-8: #{e.message}"
+      end
     end
 
     # A field declared on a Thrift::Struct subclass
@@ -399,14 +491,14 @@ module Herringbone
     #   field 1, :name, :i32
     class Struct
       class << self
-        # @return [Array<Field>] declared fields, sorted by id
+        # @return [Array<Field>] declared fields, sorted by id, including the superclass's
         def fields
-          @fields || []
+          @fields || ((superclass < Struct) ? superclass.fields : [])
         end
 
         # @return [Hash{Integer => Field}] declared fields by id, for decoding
         def fields_by_id
-          @fields_by_id || {}
+          @fields_by_id || ((superclass < Struct) ? superclass.fields_by_id : {})
         end
 
         # Declares a field and defines its accessor
@@ -475,6 +567,12 @@ module Herringbone
       def ==(other)
         other.class == self.class && other.to_h == to_h
       end
+      alias_method :eql?, :==
+
+      # @return [Integer] hash agreeing with #==, so equal structs work as Hash keys and in +uniq+
+      def hash
+        [self.class, to_h].hash
+      end
 
       # @return [String] short class name and the set fields
       def inspect

data/lib/herringbone/types.rb

@@ -18,6 +18,8 @@ module Herringbone
   #   DECIMAL               <-> BigDecimal
   #   UUID                  <-> String "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
   #   FLOAT16               <-> Float
+  #
+  # @api private
   module Types
     module_function
 

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/herringbone/visualizer.rb

@@ -15,6 +15,8 @@ module Herringbone
   # Used through Inspector#to_html:
   #
   #   File.open("data.parquet", "rb") { |io| Herringbone::Inspector.new(io).to_html }
+  #
+  # @api private
   class Visualizer
     # highlight.js build loaded by the page to colour the footer JSON; optional.
     HIGHLIGHT_JS = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"

data/lib/herringbone/writer.rb

@@ -76,6 +76,8 @@ module Herringbone
     MAX_DICTIONARY_BYTES = 1024 * 1024
     # The row group byte size is first estimated after this many rows, then after every row group
     ESTIMATE_AFTER_ROWS = 1000
+    # Options .open_for_copies refuses, since the row groups it writes keep their boundaries
+    ROW_GROUP_OPTIONS = %i[row_group_bytes row_group_rows].freeze
 
     # @return [Schema] schema the rows are written with
     attr_reader :schema
@@ -121,6 +123,29 @@ module Herringbone
       result
     end
 
+    # Internal (used by Redaction and Combiner): .open for a file whose row groups are all written
+    # with #write_row_group, one for each row group of another file, so they keep their boundaries
+    #
+    # @param io [IO, #write] destination
+    # @param schema [Schema] schema of the rows
+    # @param refusal [String] why +row_group_bytes:+ and +row_group_rows:+ cannot be given, e.g.
+    #   "a redaction keeps the row groups of the input"
+    # @param options [Hash{Symbol => Object}] see .open, without the row group options
+    # @option options [Symbol] :compression (:snappy) codec of the chunks encoded again
+    # @option options [Hash{String => String}] :metadata ({}) key/value metadata for the footer
+    # @option options [EncryptionConfiguration, Hash{Symbol => Object}, nil] :encryption (nil) see #initialize
+    # @yield [writer] the open writer
+    # @yieldparam writer [Writer] writer to write row groups to
+    # @yieldreturn [Object] returned by open_for_copies
+    # @return [Object] the block's value
+    # @raise [ArgumentError] for +row_group_bytes:+ / +row_group_rows:+, or an invalid option
+    # @api private
+    def self.open_for_copies(io, schema, refusal, **options, &block)
+      bad = options.keys & ROW_GROUP_OPTIONS
+      raise ArgumentError, "#{bad.join(", ")}: #{refusal}" unless bad.empty?
+      self.open(io, schema, row_group_bytes: 1 << 62, **options, &block)
+    end
+
     # Validates the options and writes the leading magic bytes to +io+.
     #
     # @param io [IO, #write] destination; switched to binmode when it supports that
@@ -269,11 +294,12 @@ module Herringbone
     #   @return [Format::OffsetIndex, nil] the chunk's OffsetIndex, with source file offsets
     # @!attribute bloom_filter
     #   @return [String, nil] the chunk's encoded bloom filter, header included
+    # @api private
     CopiedChunk = Struct.new(:chunk, :start, :bytes, :column_index, :offset_index, :bloom_filter)
 
-    # Internal (used by Redaction): writes a row group of +num_rows+ rows from the buffered values,
-    # except for the columns in +copies+, whose chunks are copied byte for byte from another file
-    # with their offsets rebased. Starts a new row group afterwards.
+    # Internal (used by Redaction and Combiner): writes a row group of +num_rows+ rows from the
+    # buffered values, except for the columns in +copies+, whose chunks are copied byte for byte
+    # from another file with their offsets rebased. Starts a new row group afterwards.
     #
     # @param num_rows [Integer] rows in the row group
     # @param copies [Hash{Integer => CopiedChunk}] column index => chunk to copy instead of encoding
@@ -285,6 +311,7 @@ module Herringbone
     # @param sorting_columns [Array<Format::SortingColumn>, nil] stored on the RowGroup as given
     # @return [void]
     # @raise [Error] when the writer is closed
+    # @api private
     def write_row_group(num_rows, copies: {}, codecs: {}, bloom_filters: {}, sorting_columns: nil)
       raise Error, "Writer is closed" if @closed
       if @encryption && (encrypted = copies.keys.find { |i| @encryption.encrypted?(i) })
@@ -293,7 +320,7 @@ module Herringbone
       start = @pos
       chunks = @schema.columns.map do |col|
         if (copy = copies[col.index])
-          copy_column_chunk(copy)
+          copy_column_chunk(col, copy)
         else
           bloom = @bloom_filters[col.dotted_path]
           bloom ||= bloom_filter_settings({}) if bloom_filters[col.index] && BloomFilter::TYPES.include?(col.type)
@@ -314,21 +341,23 @@ module Herringbone
       reset_buffers
     end
 
-    # Internal (used by Redaction): whether the column is encrypted in this file
+    # Internal (used by Redaction and Combiner): whether the column is encrypted in this file
     #
     # @param index [Integer] leaf column index
     # @return [Boolean]
+    # @api private
     def encrypted_column?(index) = @encryption&.encrypted?(index) || false
 
-    # Internal (used by Redaction): shreds one value per row of a top-level field into the buffers,
-    # for #write_row_group. Other fields are left as they are, so the caller decides which columns
-    # are encoded and which are copied.
+    # Internal (used by Redaction and Combiner): shreds one value per row of a top-level field into
+    # the buffers, for #write_row_group. Other fields are left as they are, so the caller decides
+    # which columns are encoded and which are copied.
     #
     # @param name [String] top-level field name
     # @param values [Array] the field's value for each row
     # @return [void]
     # @raise [ArgumentError] when the schema has no such field
     # @raise [EncodeError] when a value cannot be written to the field
+    # @api private
     def buffer_field(name, values)
       field = @schema.field(name) or raise ArgumentError, "No such field #{name.inspect}"
       values.each_with_index do |value, i|
@@ -414,6 +443,7 @@ module Herringbone
     #   @return [String, Array<Integer>, nil] repetition levels, like +defs+; nil for non-repeated columns
     # @!attribute values
     #   @return [Array, ByteValues] non-null physical values
+    # @api private
     ColumnBuffer = Struct.new(:defs, :reps, :values)
 
     # Starts a new row group: fresh buffers per column, and the per-field write plan
@@ -769,6 +799,7 @@ module Herringbone
     #   @return [Integer] non-null values in the page
     # @!attribute range
     #   @return [Array(Object, Object), nil] [min, max] physical values, nil when there are none
+    # @api private
     PageInfo = Struct.new(:offset, :size, :first_row, :nulls, :non_null, :range)
 
     # @param pages [Array<PageInfo>] pages of a column chunk
@@ -781,14 +812,17 @@ module Herringbone
 
     # Writes a chunk copied from another file. The pages, the ColumnIndex and the bloom filter
     # hold no file offsets and go out as they are; the ColumnMetaData and the OffsetIndex do, so
-    # those are rebased onto where the chunk lands in this file.
+    # those are rebased onto where the chunk lands in this file. The path is this file's, which
+    # differs when the source file names list elements another way ("list.item", "array").
     #
+    # @param col [Schema::Column] the column the chunk is copied into
     # @param copy [CopiedChunk] the chunk to copy
     # @return [Format::ColumnChunk] chunk with its ColumnMetaData, for the row group
-    def copy_column_chunk(copy)
+    def copy_column_chunk(col, copy)
       start = @pos
       shift = start - copy.start
       meta = Format::ColumnMetaData.decode(copy.chunk.meta_data.encode).first
+      meta.path_in_schema = col.path
       meta.data_page_offset += shift
       dict = meta.dictionary_page_offset
       # Some writers store 0 when there is no dictionary page
@@ -1070,7 +1104,7 @@ module Herringbone
     # keeps in their source codec use that codec's default
     #
     # @param codec [Integer] codec id
-    # @param data [String] bytes to compress
+    # @param data [String, Array<String>] bytes to compress, or the parts of them
     # @return [String] compressed bytes
     def compress(codec, data)
       Compression.compress(codec, data, (codec == @codec) ? @compression_level : nil)
@@ -1080,28 +1114,30 @@ module Herringbone
     # stored, encrypted.
     #
     # @param header [Format::PageHeader] header; sizes and CRC32 are filled in here
-    # @param body [String] uncompressed page body
-    # @param compressed [String, nil] bytes to write as the page body, when already prepared (v2 data
-    #   pages, whose levels stay uncompressed); +body+ is compressed otherwise
+    # @param body [String, Array<String>] uncompressed page body, or its parts
+    # @param compressed [String, Array<String>, nil] bytes to write as the page body, or their
+    #   parts, when already prepared (v2 data pages, whose levels stay uncompressed); +body+ is
+    #   compressed otherwise
     # @param codec [Integer] codec id to compress +body+ with
     # @param crypto [Encryption::ModuleCrypto, nil] encryption of the column's modules
     # @param ordinal [Integer, nil] data page ordinal within the chunk; nil for a dictionary page
     # @return [Integer]
     def write_page(header, body, compressed = nil, codec: @codec, crypto: nil, ordinal: nil)
-      compressed ||= compress(codec, body)
-      header.uncompressed_page_size ||= body.bytesize
+      parts = Array(compressed || compress(codec, body))
+      header.uncompressed_page_size ||= Array(body).sum(&:bytesize)
       dictionary = header.type == Format::PageType::DICTIONARY_PAGE
       if crypto
-        compressed = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE, compressed, ordinal)
+        parts = [crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE : Encryption::DATA_PAGE, parts.join, ordinal)]
       end
-      header.compressed_page_size = compressed.bytesize
-      header.crc = Zlib.crc32(compressed).then { |c| (c >= 0x8000_0000) ? c - 0x1_0000_0000 : c }
+      header.compressed_page_size = parts.sum(&:bytesize)
+      crc = parts.inject(0) { |c, part| Zlib.crc32(part, c) }
+      header.crc = (crc >= 0x8000_0000) ? crc - 0x1_0000_0000 : crc
       encoded = header.encode
       if crypto
         encoded = crypto.encrypt(dictionary ? Encryption::DICTIONARY_PAGE_HEADER : Encryption::DATA_PAGE_HEADER, encoded, ordinal)
       end
       write_raw(encoded)
-      write_raw(compressed)
+      parts.each { |part| write_raw(part) unless part.empty? }
       encoded.bytesize + header.uncompressed_page_size
     end
 
@@ -1117,7 +1153,7 @@ module Herringbone
     #   modules and the page's ordinal, for an encrypted column
     # @return [Integer] uncompressed size including the header
     def write_data_page_v1(n, rep_bytes, def_bytes, encoded, encoding, codec, page_crypto)
-      body = String.new(encoding: Encoding::BINARY)
+      body = []
       body << [rep_bytes.bytesize].pack("V") << rep_bytes unless rep_bytes.empty?
       body << [def_bytes.bytesize].pack("V") << def_bytes unless def_bytes.empty?
       body << encoded
@@ -1156,7 +1192,7 @@ module Herringbone
           is_compressed: codec != Format::Codec::UNCOMPRESSED
         )
       )
-      write_page(header, "".b, rep_bytes + def_bytes + compressed, codec: codec, crypto: page_crypto&.first,
+      write_page(header, "".b, [rep_bytes, def_bytes, compressed], codec: codec, crypto: page_crypto&.first,
         ordinal: page_crypto&.last)
     end
 

data/lib/herringbone/xxhash.rb

@@ -14,6 +14,8 @@ module Herringbone
   # allocations trigger so many garbage collections that hashing becomes up to 10x slower. The
   # arithmetic is written once with the small code generators below, whose output is inlined into
   # the hashing methods: no method calls or allocations in the hot paths.
+  #
+  # @api private
   module XXHash
     # 64-bit mask
     M = 0xFFFF_FFFF_FFFF_FFFF