herringbone 0.1.0 → 0.3.0

data/lib/herringbone/codecs/snappy.rb

@@ -1,33 +1,103 @@
 # frozen_string_literal: true
 
 module Herringbone
+  # Pure-Ruby compression codecs for Parquet pages: Snappy, and LZ4 (raw blocks and the Hadoop framing).
+  # Compression dispatches to these; GZIP uses Zlib and ZSTD/BROTLI use external gems.
   module Codecs
     # Pure-Ruby implementation of the raw Snappy block format (as used by Parquet),
     # see https://github.com/google/snappy/blob/main/format_description.txt
     #
+    # When the `snappy` gem (a binding to Google's libsnappy) can be loaded, it is used instead:
+    # 12x faster decompression and 27x faster compression. It is optional and only a speedup: if
+    # it is missing, the pure-Ruby code below is used silently. Both produce raw Snappy blocks the
+    # other reads.
+    #
     # 32-bit loads are done with four getbyte calls rather than unpack1(offset:) to stay
     # compatible with Ruby 3.0 - the speed difference on MRI is marginal.
     module Snappy
+      # Raised for corrupt or truncated Snappy input, and for input too large to compress.
       class Error < StandardError; end
 
+      # The compressor works on independent 64 KiB fragments, as the reference implementation does.
       BLOCK_SIZE = 1 << 16
+      # Size of the match-finder hash table, in bits (16384 entries).
       HASH_BITS = 14
+      # Right shift that keeps the top HASH_BITS bits of the 32-bit multiplicative hash.
       HASH_SHIFT = 32 - HASH_BITS
+      # Multiplier of the reference implementation's 4-byte hash.
       HASH_MUL = 0x1e35a7bd
       INPUT_MARGIN = 15 # bytes at the block end never searched for matches, as in the reference
+      # The length preamble is a 32-bit varint, so no block may decompress to more than this.
       MAX_UNCOMPRESSED = (1 << 32) - 1
 
+      # Name of the optional native gem, passed to +require+.
+      NATIVE_GEM = "snappy"
+
       module_function
 
       # @param input [String] raw snappy block
       # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the block is corrupt or truncated
       def decompress(input)
-        src = input.encoding == Encoding::BINARY ? input : input.b
+        src = (input.encoding == Encoding::BINARY) ? input : input.b
+        if (lib = native)
+          begin
+            return lib.inflate(src)
+          rescue lib::Error => e
+            raise Error, "corrupt snappy data (#{e.message})"
+          end
+        end
         return decompress_io_buffer(src) if IOBufferSupport::AVAILABLE
         decompress_string(src)
       end
 
+      # The backend in use: :native (the snappy gem) or :ruby
+      # @return [Symbol] +:native+ or +:ruby+
+      def backend
+        native ? :native : :ruby
+      end
+
+      # For tests and benchmarks: :ruby forces pure Ruby, :native requires the snappy gem
+      # (UnsupportedError if it cannot be loaded), nil goes back to the default
+      # @param name [Symbol, nil] +:ruby+, +:native+ or nil
+      # @return [void]
+      # @raise [UnsupportedError] if +:native+ is requested and the gem cannot be loaded
+      # @raise [ArgumentError] for any other backend name
+      def backend=(name)
+        @native = case name
+        when :ruby then false
+        when :native
+          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem could not be loaded")
+        when nil then nil
+        else raise ArgumentError, "Unknown Snappy backend #{name.inspect} (expected :ruby, :native or nil)"
+        end
+      end
+
+      # The native library to use, resolved (and memoized) on first call unless forced by backend=.
+      # @return [Module, nil] the +::Snappy+ module, or nil when the pure-Ruby code should run
+      def native
+        @native = native_library || false if @native.nil?
+        @native || nil
+      end
+
+      # Tries to load the snappy gem once and memoizes the result.
+      # @return [Module, false] the +::Snappy+ module, or false if it is missing or lacks inflate/deflate
+      def native_library
+        if @native_lib.nil?
+          @native_lib = begin
+            require NATIVE_GEM
+            (::Snappy.respond_to?(:inflate) && ::Snappy.respond_to?(:deflate)) ? ::Snappy : false
+          rescue LoadError
+            false
+          end
+        end
+        @native_lib
+      end
+
       # Decompresses into a preallocated IO::Buffer: copies do not allocate intermediate Strings.
+      # @param src [String] raw snappy block in ASCII-8BIT
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the block is corrupt or truncated
       def decompress_io_buffer(src)
         n = src.bytesize
         expected, pos = read_varint(src, n)
@@ -98,6 +168,10 @@ module Herringbone
         inbuf&.free
       end
 
+      # Decompresses by appending to a String; the fallback when IO::Buffer is not available.
+      # @param src [String] raw snappy block in ASCII-8BIT
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the block is corrupt or truncated
       def decompress_string(src)
         n = src.bytesize
         expected, pos = read_varint(src, n)
@@ -162,10 +236,15 @@ module Herringbone
 
       # @param input [String] bytes to compress
       # @return [String] raw snappy block in ASCII-8BIT
+      # @raise [Error] if the input is larger than MAX_UNCOMPRESSED
       def compress(input)
-        src = input.encoding == Encoding::BINARY ? input : input.b
+        src = (input.encoding == Encoding::BINARY) ? input : input.b
         n = src.bytesize
+        # Checked for libsnappy too: it would truncate the 32-bit length preamble without complaint
         raise Error, "input too large for snappy" if n > MAX_UNCOMPRESSED
+        if (lib = native)
+          return lib.deflate(src)
+        end
 
         out = String.new(capacity: 32 + n + n / 6, encoding: Encoding::BINARY)
         write_varint(out, n)
@@ -181,6 +260,11 @@ module Herringbone
         out
       end
 
+      # Reads the uncompressed-length preamble (a little-endian base-128 varint) at the start of +src+.
+      # @param src [String] raw snappy block
+      # @param n [Integer] byte size of +src+
+      # @return [Array(Integer, Integer)] the declared uncompressed length and the offset just past the varint
+      # @raise [Error] if the varint is truncated, longer than 5 bytes or exceeds MAX_UNCOMPRESSED
       def read_varint(src, n)
         value = 0
         shift = 0
@@ -198,6 +282,10 @@ module Herringbone
         [value, pos]
       end
 
+      # Appends +value+ as a little-endian base-128 varint (the length preamble).
+      # @param out [String] binary output buffer, appended to
+      # @param value [Integer] non-negative length to encode
+      # @return [String] +out+
       def write_varint(out, value)
         while value >= 0x80
           out << ((value & 0x7f) | 0x80)
@@ -210,6 +298,12 @@ module Herringbone
       # positions relative to `base`; matches never cross the block boundary.
       # The 4-byte little-endian word at every position of the block is unpacked up front
       # (in C, via String#unpack) so hashing and match checks are single Array lookups.
+      # @param src [String] whole input in ASCII-8BIT
+      # @param base [Integer] offset of the fragment in +src+
+      # @param len [Integer] fragment length, at most BLOCK_SIZE
+      # @param out [String] binary output buffer, appended to
+      # @param table [Array<Integer>] zeroed hash table of 1 << HASH_BITS fragment-relative positions
+      # @return [void]
       def compress_block(src, base, len, out, table)
         ip_end = base + len
         next_emit = base
@@ -280,12 +374,24 @@ module Herringbone
 
       # words[k][j] is the 4-byte little-endian value at base + 4 * j + k, so the word at
       # relative position i is words[i & 3][i >> 2]
+      # @param src [String] whole input in ASCII-8BIT
+      # @param base [Integer] offset of the fragment in +src+
+      # @param len [Integer] fragment length
+      # @return [Array<Array<Integer>>] four arrays of 32-bit words, one per byte phase
       def block_words(src, base, len)
         Array.new(4) { |k| src.byteslice(base + k, len - k).unpack("V*") }
       end
 
       # Like match_length, but compares 4 bytes at a time using the unpacked block words,
       # which avoids allocating substrings for the (common) short matches.
+      # Falls back to match_length once a match reaches 64 bytes.
+      # @param words [Array<Array<Integer>>] fragment words from block_words
+      # @param src [String] whole input in ASCII-8BIT
+      # @param base [Integer] offset of the fragment in +src+
+      # @param s1 [Integer] absolute offset of the earlier occurrence
+      # @param s2 [Integer] absolute offset of the current position (s1 < s2)
+      # @param limit [Integer] absolute offset not to read past (the fragment end)
+      # @return [Integer] number of equal bytes starting at s1 and s2
       def match_length_words(words, src, base, s1, s2, limit)
         start = s2
         r1 = s1 - base
@@ -307,6 +413,11 @@ module Herringbone
 
       # Number of equal bytes at s1 and s2 (s1 < s2), not reading past limit. Gallops
       # with byteslice comparisons (memcmp) to avoid per-byte loops on long matches.
+      # @param src [String] whole input in ASCII-8BIT
+      # @param s1 [Integer] absolute offset of the earlier occurrence
+      # @param s2 [Integer] absolute offset of the current position
+      # @param limit [Integer] absolute offset not to read past
+      # @return [Integer] number of equal bytes
       def match_length(src, s1, s2, limit)
         start = s2
         return 0 if s2 >= limit || src.getbyte(s1) != src.getbyte(s2)
@@ -330,6 +441,13 @@ module Herringbone
         s2 - start
       end
 
+      # Appends a literal element: the tag (with a 1-4 byte length extension for long literals)
+      # followed by the bytes themselves. Does nothing for an empty literal.
+      # @param out [String] binary output buffer, appended to
+      # @param src [String] whole input in ASCII-8BIT
+      # @param pos [Integer] offset of the literal bytes in +src+
+      # @param len [Integer] number of literal bytes
+      # @return [String, nil] +out+, or nil when +len+ is zero
       def emit_literal(out, src, pos, len)
         return if len == 0
         n = len - 1
@@ -348,6 +466,11 @@ module Herringbone
       end
 
       # Offsets are always < 64KB (matches stay within a block), so 4-byte offsets are never needed.
+      # Long matches are split into copies of at most 64 bytes, never leaving a remainder under 4.
+      # @param out [String] binary output buffer, appended to
+      # @param offset [Integer] backward distance to the match source, 1...65536
+      # @param len [Integer] match length, at least 4
+      # @return [String] +out+
       def emit_copy(out, offset, len)
         while len >= 68
           emit_copy_upto64(out, offset, 64)
@@ -360,6 +483,12 @@ module Herringbone
         emit_copy_upto64(out, offset, len)
       end
 
+      # Appends one copy element: the 2-byte form (1-byte offset) when len is 4..11 and offset < 2048,
+      # otherwise the 3-byte form with a 2-byte offset.
+      # @param out [String] binary output buffer, appended to
+      # @param offset [Integer] backward distance to the match source, 1...65536
+      # @param len [Integer] copy length, 4..64
+      # @return [String] +out+
       def emit_copy_upto64(out, offset, len)
         if len < 12 && offset < 2048
           out << (1 | ((len - 4) << 2) | ((offset >> 8) << 5)) << (offset & 0xff)
@@ -368,7 +497,7 @@ module Herringbone
         end
       end
 
-      private_class_method :decompress_io_buffer, :decompress_string, :read_varint, :write_varint, :compress_block, :block_words, :match_length_words,
+      private_class_method :native, :native_library, :decompress_io_buffer, :decompress_string, :read_varint, :write_varint, :compress_block, :block_words, :match_length_words,
         :match_length, :emit_literal, :emit_copy, :emit_copy_upto64
     end
   end

data/lib/herringbone/compression.rb

@@ -2,27 +2,121 @@
 
 require "zlib"
 require "stringio"
-require "zstd-ruby"
-require "brotli"
 
 module Herringbone
-  # Dispatches page (de)compression by Parquet codec id. Snappy and LZ4 are pure Ruby,
-  # GZIP uses zlib, ZSTD and Brotli use the zstd-ruby and brotli gems.
+  # Raised when a file uses (or a writer asks for) a codec whose library is not installed
+  class MissingCodecError < UnsupportedError
+    # @return [String] codec name (+"ZSTD"+) and name of the gem providing it (+"zstd-ruby"+)
+    attr_reader :codec, :gem_name
+
+    # @param codec [String] codec name, as shown in the message
+    # @param gem_name [String] gem to add to the Gemfile
+    # @param load_error [LoadError] the error from requiring the gem, whose message is included
+    def initialize(codec, gem_name, load_error)
+      @codec = codec
+      @gem_name = gem_name
+      super("#{codec} compression needs the \"#{gem_name}\" gem, which could not be loaded " \
+        "(#{load_error.message}). Add `gem \"#{gem_name}\"` to your Gemfile to use #{codec}.")
+    end
+  end
+
+  # Dispatches page (de)compression by Parquet codec id. Snappy and LZ4 are pure Ruby and GZIP
+  # uses zlib, so those always work. ZSTD and Brotli come from optional gems (zstd-ruby, brotli),
+  # which are required on first use; if they are missing, MissingCodecError says what to add.
+  # (Snappy also uses the optional snappy gem when it is installed, see Codecs::Snappy.)
   module Compression
     module_function
 
-    CODECS_BY_NAME = {
-      none: Format::Codec::UNCOMPRESSED, uncompressed: Format::Codec::UNCOMPRESSED,
-      snappy: Format::Codec::SNAPPY, gzip: Format::Codec::GZIP, brotli: Format::Codec::BROTLI,
-      lz4: Format::Codec::LZ4_RAW, lz4_raw: Format::Codec::LZ4_RAW, lz4_hadoop: Format::Codec::LZ4,
-      zstd: Format::Codec::ZSTD
+    # Codecs backed by optional native gems: codec id => [name, gem, require path, constant]
+    LIBRARIES = {
+      Format::Codec::ZSTD => ["ZSTD", "zstd-ruby", "zstd-ruby", :Zstd],
+      Format::Codec::BROTLI => ["Brotli", "brotli", "brotli", :Brotli]
+    }.freeze
+
+    @libraries = {}
+    @library_mutex = Mutex.new
+
+    # The library module for a codec backed by an optional gem, requiring it on first use
+    #
+    # @param codec [Integer] a codec id that is a key of LIBRARIES
+    # @return [Module] the gem's module (+Zstd+ or +Brotli+)
+    # @raise [MissingCodecError] when the gem cannot be loaded
+    # @raise [KeyError] for a codec id not in LIBRARIES
+    def library(codec)
+      @libraries.fetch(codec) do
+        @library_mutex.synchronize do
+          @libraries.fetch(codec) do
+            name, gem_name, path, const = LIBRARIES.fetch(codec)
+            begin
+              require_library(path)
+            rescue LoadError => e
+              raise MissingCodecError.new(name, gem_name, e)
+            end
+            @libraries[codec] = Object.const_get(const)
+          end
+        end
+      end
+    end
+
+    # Requires a codec gem; a separate method so tests can stub it to simulate a missing gem
+    #
+    # @param path [String] require path of the gem
+    # @return [Boolean] whether the library was newly loaded
+    # @raise [LoadError] when the gem is not installed
+    def require_library(path)
+      require path
+    end
+
+    # Raises MissingCodecError (or UnsupportedError) unless +codec+ can be used
+    #
+    # @param codec [Integer, Symbol, String] codec id or name (see NAMES)
+    # @return [Module, nil] the gem's module for a gem-backed codec, nil for a built-in one
+    # @raise [MissingCodecError] when the codec's gem cannot be loaded
+    # @raise [UnsupportedError] for a codec herringbone does not implement (LZO)
+    # @raise [ArgumentError] for an unknown codec name
+    def ensure_available!(codec)
+      codec = codec_id(codec)
+      return library(codec) if LIBRARIES.key?(codec)
+      return if SUPPORTED.include?(codec)
+      raise UnsupportedError, "#{Format::Codec::NAMES[codec] || codec} compression is not supported"
+    end
+
+    # Codec ids that work without optional gems
+    SUPPORTED = [
+      Format::Codec::UNCOMPRESSED, Format::Codec::SNAPPY, Format::Codec::GZIP,
+      Format::Codec::LZ4_RAW, Format::Codec::LZ4
+    ].freeze
+
+    # Codec id => name, as given to the writer's compression: option and listed by Herringbone.codecs
+    NAMES = {
+      Format::Codec::UNCOMPRESSED => :none, Format::Codec::SNAPPY => :snappy, Format::Codec::GZIP => :gzip,
+      Format::Codec::LZ4_RAW => :lz4, Format::Codec::LZ4 => :lz4_hadoop, Format::Codec::ZSTD => :zstd,
+      Format::Codec::BROTLI => :brotli, Format::Codec::LZO => :lzo
     }.freeze
+    # Codec name => codec id, the inverse of NAMES
+    CODECS_BY_NAME = NAMES.invert.freeze
 
+    # Codec id for a codec name; Integers are taken to be ids already and returned unchecked
+    #
+    # @param name [Integer, Symbol, String] codec id, or a name from NAMES (case-insensitive)
+    # @return [Integer] the Format::Codec id
+    # @raise [ArgumentError] for an unknown name
     def codec_id(name)
       return name if name.is_a?(Integer)
-      CODECS_BY_NAME.fetch(name.to_s.downcase.to_sym) { raise ArgumentError, "Unknown compression codec #{name.inspect}" }
+      CODECS_BY_NAME.fetch(name.to_s.downcase.to_sym) do
+        raise ArgumentError, "Unknown compression codec #{name.inspect}, expected one of #{NAMES.values.map(&:inspect).join(", ")}"
+      end
     end
 
+    # Decompresses a page body, checking the result against the size the page header declares
+    #
+    # @param codec [Integer] Format::Codec id from the column chunk metadata
+    # @param data [String] compressed bytes
+    # @param uncompressed_size [Integer] expected decompressed size in bytes
+    # @return [String] decompressed bytes (binary)
+    # @raise [UnsupportedError] for a codec herringbone does not implement
+    # @raise [MissingCodecError] when the codec's gem cannot be loaded
+    # @raise [FormatError] when the decompressed size does not match +uncompressed_size+
     def decompress(codec, data, uncompressed_size)
       return "".b if uncompressed_size.zero? && data.empty?
       out = case codec
@@ -31,18 +125,25 @@ module Herringbone
       when Format::Codec::GZIP then gunzip(data)
       when Format::Codec::LZ4_RAW then Codecs::LZ4.decompress_block(data, uncompressed_size)
       when Format::Codec::LZ4 then Codecs::LZ4.decompress_hadoop(data, uncompressed_size)
-      when Format::Codec::ZSTD then ::Zstd.decompress(data)
-      when Format::Codec::BROTLI then ::Brotli.inflate(data)
+      when Format::Codec::ZSTD then library(codec).decompress(data)
+      when Format::Codec::BROTLI then library(codec).inflate(data)
       else
         raise UnsupportedError, "Unsupported compression codec #{Format::Codec::NAMES[codec] || codec}"
       end
       out = out.b
       if out.bytesize != uncompressed_size
-        raise DecodeError, "Decompressed #{out.bytesize} bytes, expected #{uncompressed_size}"
+        raise FormatError, "Decompressed #{out.bytesize} bytes, expected #{uncompressed_size}"
       end
       out
     end
 
+    # Compresses a page body
+    #
+    # @param codec [Integer] Format::Codec id
+    # @param data [String] bytes to compress
+    # @return [String] compressed bytes (binary)
+    # @raise [UnsupportedError] for a codec herringbone does not implement
+    # @raise [MissingCodecError] when the codec's gem cannot be loaded
     def compress(codec, data)
       case codec
       when Format::Codec::UNCOMPRESSED then data
@@ -50,18 +151,22 @@ module Herringbone
       when Format::Codec::GZIP then Zlib.gzip(data)
       when Format::Codec::LZ4_RAW then Codecs::LZ4.compress_block(data)
       when Format::Codec::LZ4 then Codecs::LZ4.compress_hadoop(data)
-      when Format::Codec::ZSTD then ::Zstd.compress(data)
-      when Format::Codec::BROTLI then ::Brotli.deflate(data)
+      when Format::Codec::ZSTD then library(codec).compress(data)
+      when Format::Codec::BROTLI then library(codec).deflate(data)
       else
         raise UnsupportedError, "Unsupported compression codec #{Format::Codec::NAMES[codec] || codec}"
       end.b
     end
 
     # Handles files whose gzip data consists of several concatenated members
+    #
+    # @param data [String] gzip data, one or more members
+    # @return [String] the members' decompressed bytes concatenated (binary)
+    # @raise [Zlib::GzipFile::Error] on corrupt gzip data
     def gunzip(data)
       io = StringIO.new(data)
       out = String.new(encoding: Encoding::BINARY)
-      loop do
+      while true
         gz = Zlib::GzipReader.new(io)
         out << gz.read
         unused = gz.unused

data/lib/herringbone/encodings/delta.rb

@@ -6,56 +6,84 @@ module Herringbone
     module Delta
       module_function
 
+      # Values per block written by the encoder (the size parquet-mr and Arrow use).
       BLOCK_SIZE = 128
+      # Miniblocks per block written by the encoder, so 32 values per miniblock.
       MINIBLOCKS = 4
 
+      # @param n [Integer] zigzag-encoded unsigned integer
+      # @return [Integer] the signed integer it represents
       def zigzag_decode(n) = (n >> 1) ^ -(n & 1)
+
+      # @param n [Integer] signed integer
+      # @return [Integer] zigzag-encoded form (0, -1, 1, -2 map to 0, 1, 2, 3)
       def zigzag_encode(n) = n.negative? ? ((-n) << 1) - 1 : n << 1
 
       # Wraps an integer into the signed range of +bits+ bits
+      # @param v [Integer] integer to wrap
+      # @param bits [Integer] target width, 32 or 64
+      # @return [Integer] +v+ modulo 2^bits, as a two's complement signed integer
       def wrap(v, bits)
         half = 1 << (bits - 1)
         ((v + half) & ((1 << bits) - 1)) - half
       end
 
       # Decodes DELTA_BINARY_PACKED integers. +bits+ is 32 or 64 (for wraparound).
-      # Returns [values, new_pos]. If +count+ is nil, the total from the header is used.
+      # Returns [values, new_pos]. If +count+ is nil, the total from the header is used. A smaller
+      # +count+ decodes only that many values, but the returned offset is still the end of the
+      # whole encoded block, so data following it can be read from there.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the block header
+      # @param bits [Integer] integer width that deltas wrap around in, 32 or 64
+      # @param count [Integer, nil] maximum number of values to decode
+      # @return [Array(Array<Integer>, Integer)] the decoded values and the offset just past the encoded block
+      # @raise [FormatError] on an invalid header or miniblock bit width, or truncated data
       def decode_binary_packed(data, pos, bits = 64, count = nil)
         block_size, pos = RLE.read_uleb(data, pos)
         miniblocks, pos = RLE.read_uleb(data, pos)
         total, pos = RLE.read_uleb(data, pos)
         first, pos = RLE.read_uleb(data, pos)
-        raise DecodeError, "Invalid DELTA_BINARY_PACKED header" if miniblocks.zero? || block_size % miniblocks != 0
+        raise FormatError, "Invalid DELTA_BINARY_PACKED header" if miniblocks.zero? || block_size % miniblocks != 0
         per_mini = block_size / miniblocks
-        raise DecodeError, "Invalid miniblock size #{per_mini}" if per_mini % 8 != 0
-        total = count if count && count < total
+        raise FormatError, "Invalid miniblock size #{per_mini}" if per_mini % 8 != 0
+        want = (count && count < total) ? count : total
         values = []
         return [values, pos] if total.zero?
         last = zigzag_decode(first)
-        values << last
+        values << last if want.positive?
         half = 1 << (bits - 1)
         mask = (1 << bits) - 1
-        while values.size < total
+        left = total - 1 # deltas still encoded, decoded or skipped
+        while left.positive?
           min_delta, pos = RLE.read_uleb(data, pos)
           min_delta = zigzag_decode(min_delta)
           widths = data.byteslice(pos, miniblocks).unpack("C*")
           pos += miniblocks
           widths.each do |w|
-            break if values.size >= total
-            raise DecodeError, "Invalid delta bit width #{w}" if w > bits
-            deltas = RLE.unpack_bits(data, pos, per_mini, w)
-            pos += per_mini * w / 8
-            take = total - values.size
-            deltas = deltas.first(take) if take < per_mini
-            deltas.each do |d|
-              last = ((last + min_delta + d + half) & mask) - half
-              values << last
+            # Miniblocks past the last value have a width byte but no body
+            break unless left.positive?
+            raise FormatError, "Invalid delta bit width #{w}" if w > bits
+            if values.size < want
+              deltas = RLE.unpack_bits(data, pos, per_mini, w)
+              take = want - values.size
+              deltas = deltas.first(take) if take < per_mini
+              deltas.each do |d|
+                last = ((last + min_delta + d + half) & mask) - half
+                values << last
+              end
             end
+            pos += per_mini * w / 8
+            left -= per_mini
           end
         end
         [values, pos]
       end
 
+      # Encodes integers with DELTA_BINARY_PACKED, using BLOCK_SIZE values per block in MINIBLOCKS
+      # miniblocks. Deltas wrap at +bits+ so INT32 columns never need more than 32-bit widths.
+      # @param values [Array<Integer>] signed integers that fit in +bits+ bits
+      # @param bits [Integer] physical integer width, 32 or 64
+      # @return [String] encoded bytes in ASCII-8BIT
       def encode_binary_packed(values, bits = 64)
         out = String.new(encoding: Encoding::BINARY)
         per_mini = BLOCK_SIZE / MINIBLOCKS
@@ -82,35 +110,54 @@ module Herringbone
         out
       end
 
+      # Decodes DELTA_LENGTH_BYTE_ARRAY: DELTA_BINARY_PACKED lengths followed by the concatenated bytes.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the lengths block
+      # @param count [Integer] number of values to decode
+      # @return [Array(Array<String>, Integer)] binary slices of +data+ and the offset just past them
+      # @raise [FormatError] if there are too few lengths or a value runs past the end of +data+
       def decode_length_byte_array(data, pos, count)
         lengths, pos = decode_binary_packed(data, pos, 32, count)
-        raise DecodeError, "DELTA_LENGTH_BYTE_ARRAY has #{lengths.size} lengths, need #{count}" if lengths.size < count
+        raise FormatError, "DELTA_LENGTH_BYTE_ARRAY has #{lengths.size} lengths, need #{count}" if lengths.size < count
         out = Array.new(count)
         size = data.bytesize
         lengths.each_with_index do |len, i|
-          raise DecodeError, "DELTA_LENGTH_BYTE_ARRAY value overruns page" if len.negative? || pos + len > size
+          raise FormatError, "DELTA_LENGTH_BYTE_ARRAY value overruns page" if len.negative? || pos + len > size
           out[i] = data.byteslice(pos, len)
           pos += len
         end
         [out, pos]
       end
 
+      # Encodes byte strings with DELTA_LENGTH_BYTE_ARRAY.
+      # @param values [Array<String>] byte strings, in any encoding
+      # @return [String] encoded bytes in ASCII-8BIT
       def encode_length_byte_array(values)
         encode_binary_packed(values.map(&:bytesize), 32) << values.pack("a*" * values.size)
       end
 
+      # Decodes DELTA_BYTE_ARRAY (incremental encoding): DELTA_BINARY_PACKED prefix lengths, then
+      # the suffixes as DELTA_LENGTH_BYTE_ARRAY. Each value is the previous value's prefix plus its suffix.
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the prefix lengths block
+      # @param count [Integer] number of values to decode
+      # @return [Array(Array<String>, Integer)] binary values and the offset just past them
+      # @raise [FormatError] if a prefix is negative or longer than the previous value, or the data is malformed
       def decode_byte_array(data, pos, count)
         prefixes, pos = decode_binary_packed(data, pos, 32, count)
         suffixes, pos = decode_length_byte_array(data, pos, count)
         prev = "".b
         out = Array.new(count) do |i|
           prefix = prefixes[i]
-          raise DecodeError, "DELTA_BYTE_ARRAY prefix longer than previous value" if prefix > prev.bytesize
+          raise FormatError, "DELTA_BYTE_ARRAY prefix length #{prefix} out of range" if prefix > prev.bytesize || prefix.negative?
           prev = prefix.zero? ? suffixes[i] : prev.byteslice(0, prefix) + suffixes[i]
         end
         [out, pos]
       end
 
+      # Encodes byte strings with DELTA_BYTE_ARRAY, sharing the common byte prefix with the previous value.
+      # @param values [Array<String>] byte strings, in any encoding
+      # @return [String] encoded bytes in ASCII-8BIT
       def encode_byte_array(values)
         prev = "".b
         prefixes = []
@@ -132,13 +179,23 @@ module Herringbone
       module_function
 
       # Returns the value bytes re-interleaved into PLAIN layout
+      # @param data [String] binary page data
+      # @param pos [Integer] byte offset of the first stream
+      # @param count [Integer] number of values
+      # @param width [Integer] byte width of one value
+      # @return [Array(String, Integer)] PLAIN-layout bytes and the offset just past the streams
+      # @raise [FormatError] if fewer than count * width bytes remain
       def decode(data, pos, count, width)
         nbytes = count * width
-        raise DecodeError, "Truncated BYTE_STREAM_SPLIT data" if pos + nbytes > data.bytesize
+        raise FormatError, "Truncated BYTE_STREAM_SPLIT data" if pos + nbytes > data.bytesize
         streams = Array.new(width) { |k| data.byteslice(pos + k * count, count).unpack("C*") }
         [streams[0].zip(*streams[1..]).flatten.pack("C*"), pos + nbytes]
       end
 
+      # Splits PLAIN-layout fixed-width values into +width+ byte streams. Inverse of decode.
+      # @param plain [String] PLAIN-encoded values, a multiple of +width+ bytes
+      # @param width [Integer] byte width of one value
+      # @return [String] the concatenated streams in ASCII-8BIT
       def encode(plain, width)
         count = plain.bytesize / width
         bytes = plain.unpack("C*")