herringbone 0.2.0 → 0.4.0

data/lib/herringbone/codecs/lzo.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module Herringbone
+  module Codecs
+    # Pure-Ruby LZO1X decompressor, for reading Parquet files written by parquet-mr with
+    # hadoop-lzo. There is no compressor: nothing but hadoop-lzo writes LZO, and nothing
+    # outside the JVM reads it.
+    #
+    # Written from the bitstream description in the Linux kernel's
+    # Documentation/staging/lzo.rst, not from the (GPL) LZO sources.
+    module LZO
+      # Raised for corrupt or truncated LZO input.
+      class Error < StandardError; end
+
+      # Hadoop block header: big-endian 32-bit uncompressed size of the block.
+      HADOOP_BLOCK_PREFIX = 4
+      # Hadoop chunk header: big-endian 32-bit compressed size of the chunk.
+      HADOOP_CHUNK_PREFIX = 4
+      # Shorthand for Encoding::BINARY.
+      BINARY = Encoding::BINARY
+
+      module_function
+
+      # Parquet LZO (codec 3). hadoop-lzo writes Hadoop block framing; data that does not fit
+      # the framing is decoded as one bare LZO1X stream.
+      # @param input [String] compressed page data
+      # @param uncompressed_size [Integer] exact decompressed size, from the page header
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if neither layout decodes
+      def decompress_hadoop(input, uncompressed_size)
+        src = binary(input)
+        try_hadoop(src, uncompressed_size) || decompress_block(src, uncompressed_size)
+      end
+
+      # Decompress one bare LZO1X stream that must expand to exactly uncompressed_size bytes.
+      # @param input [String] LZO1X stream, ending with the end-of-stream marker
+      # @param uncompressed_size [Integer] exact decompressed size
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the stream is corrupt or does not decode to +uncompressed_size+ bytes
+      def decompress_block(input, uncompressed_size)
+        src = binary(input)
+        out = String.new(capacity: uncompressed_size, encoding: BINARY)
+        ip = decode_block(src, 0, src.bytesize, out, uncompressed_size)
+        raise Error, "#{src.bytesize - ip} bytes after the end of the LZO stream" unless ip == src.bytesize
+        unless out.bytesize == uncompressed_size
+          raise Error, "LZO stream decoded to #{out.bytesize} bytes, expected #{uncompressed_size}"
+        end
+        out
+      end
+
+      # @param str [String] input in any encoding
+      # @return [String] +str+ itself if already binary, otherwise a binary copy
+      def binary(str)
+        (str.encoding == BINARY) ? str : str.b
+      end
+
+      # Hadoop's BlockCompressorStream layout: blocks of (uncompressed size, then chunks of
+      # (compressed size, LZO stream) until the block is filled). Each chunk is a separate stream.
+      # @param src [String] binary input
+      # @param uncompressed_size [Integer] exact decompressed size expected over all blocks
+      # @return [String, nil] decompressed bytes, or nil if +src+ is not valid Hadoop-framed LZO
+      def try_hadoop(src, uncompressed_size)
+        n = src.bytesize
+        out = String.new(capacity: uncompressed_size, encoding: BINARY)
+        ip = 0
+        while ip < n
+          return nil if n - ip < HADOOP_BLOCK_PREFIX
+          block_end = out.bytesize + src.byteslice(ip, HADOOP_BLOCK_PREFIX).unpack1("N")
+          ip += HADOOP_BLOCK_PREFIX
+          return nil if block_end > uncompressed_size
+
+          while out.bytesize < block_end
+            return nil if n - ip < HADOOP_CHUNK_PREFIX
+            csize = src.byteslice(ip, HADOOP_CHUNK_PREFIX).unpack1("N")
+            ip += HADOOP_CHUNK_PREFIX
+            return nil if csize > n - ip
+            begin
+              return nil unless decode_block(src, ip, ip + csize, out, block_end) == ip + csize
+            rescue Error
+              return nil
+            end
+            ip += csize
+          end
+        end
+        (out.bytesize == uncompressed_size) ? out : nil
+      end
+
+      # Decode one LZO1X stream from src[ip...iend], appending to out. Matches may only reach
+      # back to where this stream's output starts. out may not grow beyond limit.
+      #
+      # Every instruction is a match optionally followed by 0-3 literals (its low two bits, the
+      # "state"), or a run of 4+ literals. What an instruction byte below 16 means depends on
+      # the state the previous instruction left behind.
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the stream in +src+
+      # @param iend [Integer] offset just past the stream
+      # @param out [String] binary output buffer, appended to
+      # @param limit [Integer] maximum total byte size of +out+
+      # @return [Integer] input offset just past the end-of-stream marker
+      # @raise [Error] if the stream is truncated, has an invalid distance or overflows +limit+
+      def decode_block(src, ip, iend, out, limit)
+        base = out.bytesize
+        raise Error, "Empty LZO stream" if ip >= iend
+
+        state = 0
+        t = src.getbyte(ip)
+        if t > 17 # the first byte may open with a literal run of its own
+          ip += 1
+          state = t - 17
+          ip = copy_literals(src, ip, iend, out, limit, state)
+          state = 4 if state > 4
+        end
+
+        while true
+          raise Error, "Truncated LZO stream" if ip >= iend
+          t = src.getbyte(ip)
+          ip += 1
+
+          if t < 16
+            if state == 0
+              len = t + 3
+              len, ip = extended_length(src, ip, iend, 18) if t == 0
+              ip = copy_literals(src, ip, iend, out, limit, len)
+              state = 4
+              next
+            end
+            raise Error, "Truncated LZO stream" if ip >= iend
+            dist = (src.getbyte(ip) << 2) + (t >> 2) + 1
+            ip += 1
+            if state == 4
+              dist += 2048
+              len = 3
+            else
+              len = 2
+            end
+            s = t & 3
+          elsif t < 32
+            len = (t & 7) + 2
+            len, ip = extended_length(src, ip, iend, 9) if t & 7 == 0
+            raise Error, "Truncated LZO stream" if ip + 2 > iend
+            le = src.getbyte(ip) | (src.getbyte(ip + 1) << 8)
+            ip += 2
+            dist = ((t & 8) << 11) + (le >> 2)
+            return ip if dist == 0 # end of stream
+            dist += 16_384
+            s = le & 3
+          elsif t < 64
+            len = (t & 31) + 2
+            len, ip = extended_length(src, ip, iend, 33) if t & 31 == 0
+            raise Error, "Truncated LZO stream" if ip + 2 > iend
+            le = src.getbyte(ip) | (src.getbyte(ip + 1) << 8)
+            ip += 2
+            dist = (le >> 2) + 1
+            s = le & 3
+          else
+            raise Error, "Truncated LZO stream" if ip >= iend
+            len = (t >> 5) + 1 # 3-4 bytes for 01LDDDSS, 5-8 for 1LLDDDSS
+            dist = (src.getbyte(ip) << 3) + ((t >> 2) & 7) + 1
+            ip += 1
+            s = t & 3
+          end
+
+          copy_match(out, base, dist, len, limit)
+          ip = copy_literals(src, ip, iend, out, limit, s) if s > 0
+          state = s
+        end
+      end
+
+      # Reads a length extension: a byte of 0 for each 255, then the nonzero remainder.
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the first extension byte
+      # @param iend [Integer] offset just past the stream
+      # @param bias [Integer] added to the extension
+      # @return [Array(Integer, Integer)] the length, and the offset just past the extension
+      # @raise [Error] if the extension runs past +iend+
+      def extended_length(src, ip, iend, bias)
+        len = bias
+        while true
+          raise Error, "Truncated LZO length" if ip >= iend
+          b = src.getbyte(ip)
+          ip += 1
+          return [len + b, ip] unless b == 0
+          len += 255
+        end
+      end
+
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the literals in +src+
+      # @param iend [Integer] offset just past the stream
+      # @param out [String] binary output buffer, appended to
+      # @param limit [Integer] maximum total byte size of +out+
+      # @param len [Integer] number of literal bytes
+      # @return [Integer] offset just past the literals
+      # @raise [Error] if the literals run past +iend+ or +out+ would exceed +limit+
+      def copy_literals(src, ip, iend, out, limit, len)
+        raise Error, "LZO literals run past end of input" if ip + len > iend
+        raise Error, "LZO output exceeds expected size" if out.bytesize + len > limit
+        out << src.byteslice(ip, len)
+        ip + len
+      end
+
+      # @param out [String] binary output buffer, appended to
+      # @param base [Integer] size of +out+ when the current stream started
+      # @param dist [Integer] how far back the match starts
+      # @param len [Integer] match length, which may exceed +dist+ (a repeating pattern)
+      # @param limit [Integer] maximum total byte size of +out+
+      # @return [String] +out+
+      # @raise [Error] if the match reaches before the stream's output or +out+ would exceed +limit+
+      def copy_match(out, base, dist, len, limit)
+        pos = out.bytesize
+        raise Error, "LZO match distance #{dist} reaches before start of output" if dist > pos - base
+        raise Error, "LZO output exceeds expected size" if pos + len > limit
+        if dist >= len
+          out << out.byteslice(pos - dist, len)
+        else
+          pattern = out.byteslice(pos - dist, dist)
+          out << (pattern * (len / dist)) << pattern.byteslice(0, len % dist)
+        end
+      end
+
+      private_class_method :binary, :try_hadoop, :decode_block, :extended_length, :copy_literals, :copy_match
+    end
+  end
+end

data/lib/herringbone/codecs/snappy.rb

@@ -1,33 +1,96 @@
 # frozen_string_literal: true
 
 module Herringbone
+  # Pure-Ruby compression codecs for Parquet pages: Snappy, and LZ4 (raw blocks and the Hadoop framing).
+  # Compression dispatches to these; GZIP uses Zlib and ZSTD/BROTLI use external gems.
   module Codecs
     # Pure-Ruby implementation of the raw Snappy block format (as used by Parquet),
     # see https://github.com/google/snappy/blob/main/format_description.txt
     #
+    # When the `snappy` gem (a binding to Google's libsnappy) is loaded, it is used instead:
+    # 12x faster decompression and 27x faster compression. It is optional and only a speedup: if
+    # it is missing, the pure-Ruby code below is used silently. Both produce raw Snappy blocks the
+    # other reads.
+    #
     # 32-bit loads are done with four getbyte calls rather than unpack1(offset:) to stay
     # compatible with Ruby 3.0 - the speed difference on MRI is marginal.
     module Snappy
+      # Raised for corrupt or truncated Snappy input, and for input too large to compress.
       class Error < StandardError; end
 
+      # The compressor works on independent 64 KiB fragments, as the reference implementation does.
       BLOCK_SIZE = 1 << 16
+      # Size of the match-finder hash table, in bits (16384 entries).
       HASH_BITS = 14
+      # Right shift that keeps the top HASH_BITS bits of the 32-bit multiplicative hash.
       HASH_SHIFT = 32 - HASH_BITS
+      # Multiplier of the reference implementation's 4-byte hash.
       HASH_MUL = 0x1e35a7bd
       INPUT_MARGIN = 15 # bytes at the block end never searched for matches, as in the reference
+      # The length preamble is a 32-bit varint, so no block may decompress to more than this.
       MAX_UNCOMPRESSED = (1 << 32) - 1
 
+      # Name of the optional native gem.
+      NATIVE_GEM = "snappy"
+
       module_function
 
       # @param input [String] raw snappy block
       # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the block is corrupt or truncated
       def decompress(input)
-        src = input.encoding == Encoding::BINARY ? input : input.b
+        src = (input.encoding == Encoding::BINARY) ? input : input.b
+        if (lib = native)
+          begin
+            return lib.inflate(src)
+          rescue lib::Error => e
+            raise Error, "corrupt snappy data (#{e.message})"
+          end
+        end
         return decompress_io_buffer(src) if IOBufferSupport::AVAILABLE
         decompress_string(src)
       end
 
+      # The backend in use: :native (the snappy gem) or :ruby
+      # @return [Symbol] +:native+ or +:ruby+
+      def backend
+        native ? :native : :ruby
+      end
+
+      # For tests and benchmarks: :ruby forces pure Ruby, :native the snappy gem
+      # (UnsupportedError if it is not loaded), nil goes back to the default
+      # @param name [Symbol, nil] +:ruby+, +:native+ or nil
+      # @return [void]
+      # @raise [UnsupportedError] if +:native+ is requested and the gem is not loaded
+      # @raise [ArgumentError] for any other backend name
+      def backend=(name)
+        @native = case name
+        when :ruby then false
+        when :native
+          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem is not loaded (require \"#{NATIVE_GEM}\")")
+        when nil then nil
+        else raise ArgumentError, "Unknown Snappy backend #{name.inspect} (expected :ruby, :native or nil)"
+        end
+      end
+
+      # The native library to use: the snappy gem when it is loaded, unless backend= forced one.
+      # @return [Module, nil] the +::Snappy+ module, or nil when the pure-Ruby code should run
+      def native
+        native = @native
+        native = native_library if native.nil?
+        native || nil
+      end
+
+      # Not memoized, so that it works the same in every Ractor and once the gem is required later.
+      # @return [Module, false] the +::Snappy+ module, or false if it is not loaded or lacks inflate/deflate
+      def native_library
+        (defined?(::Snappy.inflate) && defined?(::Snappy.deflate)) ? ::Snappy : false
+      end
+
       # Decompresses into a preallocated IO::Buffer: copies do not allocate intermediate Strings.
+      # @param src [String] raw snappy block in ASCII-8BIT
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the block is corrupt or truncated
       def decompress_io_buffer(src)
         n = src.bytesize
         expected, pos = read_varint(src, n)
@@ -98,6 +161,10 @@ module Herringbone
         inbuf&.free
       end
 
+      # Decompresses by appending to a String; the fallback when IO::Buffer is not available.
+      # @param src [String] raw snappy block in ASCII-8BIT
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the block is corrupt or truncated
       def decompress_string(src)
         n = src.bytesize
         expected, pos = read_varint(src, n)
@@ -162,10 +229,15 @@ module Herringbone
 
       # @param input [String] bytes to compress
       # @return [String] raw snappy block in ASCII-8BIT
+      # @raise [Error] if the input is larger than MAX_UNCOMPRESSED
       def compress(input)
-        src = input.encoding == Encoding::BINARY ? input : input.b
+        src = (input.encoding == Encoding::BINARY) ? input : input.b
         n = src.bytesize
+        # Checked for libsnappy too: it would truncate the 32-bit length preamble without complaint
         raise Error, "input too large for snappy" if n > MAX_UNCOMPRESSED
+        if (lib = native)
+          return lib.deflate(src)
+        end
 
         out = String.new(capacity: 32 + n + n / 6, encoding: Encoding::BINARY)
         write_varint(out, n)
@@ -181,6 +253,11 @@ module Herringbone
         out
       end
 
+      # Reads the uncompressed-length preamble (a little-endian base-128 varint) at the start of +src+.
+      # @param src [String] raw snappy block
+      # @param n [Integer] byte size of +src+
+      # @return [Array(Integer, Integer)] the declared uncompressed length and the offset just past the varint
+      # @raise [Error] if the varint is truncated, longer than 5 bytes or exceeds MAX_UNCOMPRESSED
       def read_varint(src, n)
         value = 0
         shift = 0
@@ -198,6 +275,10 @@ module Herringbone
         [value, pos]
       end
 
+      # Appends +value+ as a little-endian base-128 varint (the length preamble).
+      # @param out [String] binary output buffer, appended to
+      # @param value [Integer] non-negative length to encode
+      # @return [String] +out+
       def write_varint(out, value)
         while value >= 0x80
           out << ((value & 0x7f) | 0x80)
@@ -210,6 +291,12 @@ module Herringbone
       # positions relative to `base`; matches never cross the block boundary.
       # The 4-byte little-endian word at every position of the block is unpacked up front
       # (in C, via String#unpack) so hashing and match checks are single Array lookups.
+      # @param src [String] whole input in ASCII-8BIT
+      # @param base [Integer] offset of the fragment in +src+
+      # @param len [Integer] fragment length, at most BLOCK_SIZE
+      # @param out [String] binary output buffer, appended to
+      # @param table [Array<Integer>] zeroed hash table of 1 << HASH_BITS fragment-relative positions
+      # @return [void]
       def compress_block(src, base, len, out, table)
         ip_end = base + len
         next_emit = base
@@ -280,12 +367,24 @@ module Herringbone
 
       # words[k][j] is the 4-byte little-endian value at base + 4 * j + k, so the word at
       # relative position i is words[i & 3][i >> 2]
+      # @param src [String] whole input in ASCII-8BIT
+      # @param base [Integer] offset of the fragment in +src+
+      # @param len [Integer] fragment length
+      # @return [Array<Array<Integer>>] four arrays of 32-bit words, one per byte phase
       def block_words(src, base, len)
         Array.new(4) { |k| src.byteslice(base + k, len - k).unpack("V*") }
       end
 
       # Like match_length, but compares 4 bytes at a time using the unpacked block words,
       # which avoids allocating substrings for the (common) short matches.
+      # Falls back to match_length once a match reaches 64 bytes.
+      # @param words [Array<Array<Integer>>] fragment words from block_words
+      # @param src [String] whole input in ASCII-8BIT
+      # @param base [Integer] offset of the fragment in +src+
+      # @param s1 [Integer] absolute offset of the earlier occurrence
+      # @param s2 [Integer] absolute offset of the current position (s1 < s2)
+      # @param limit [Integer] absolute offset not to read past (the fragment end)
+      # @return [Integer] number of equal bytes starting at s1 and s2
       def match_length_words(words, src, base, s1, s2, limit)
         start = s2
         r1 = s1 - base
@@ -307,6 +406,11 @@ module Herringbone
 
       # Number of equal bytes at s1 and s2 (s1 < s2), not reading past limit. Gallops
       # with byteslice comparisons (memcmp) to avoid per-byte loops on long matches.
+      # @param src [String] whole input in ASCII-8BIT
+      # @param s1 [Integer] absolute offset of the earlier occurrence
+      # @param s2 [Integer] absolute offset of the current position
+      # @param limit [Integer] absolute offset not to read past
+      # @return [Integer] number of equal bytes
       def match_length(src, s1, s2, limit)
         start = s2
         return 0 if s2 >= limit || src.getbyte(s1) != src.getbyte(s2)
@@ -330,6 +434,13 @@ module Herringbone
         s2 - start
       end
 
+      # Appends a literal element: the tag (with a 1-4 byte length extension for long literals)
+      # followed by the bytes themselves. Does nothing for an empty literal.
+      # @param out [String] binary output buffer, appended to
+      # @param src [String] whole input in ASCII-8BIT
+      # @param pos [Integer] offset of the literal bytes in +src+
+      # @param len [Integer] number of literal bytes
+      # @return [String, nil] +out+, or nil when +len+ is zero
       def emit_literal(out, src, pos, len)
         return if len == 0
         n = len - 1
@@ -348,6 +459,11 @@ module Herringbone
       end
 
       # Offsets are always < 64KB (matches stay within a block), so 4-byte offsets are never needed.
+      # Long matches are split into copies of at most 64 bytes, never leaving a remainder under 4.
+      # @param out [String] binary output buffer, appended to
+      # @param offset [Integer] backward distance to the match source, 1...65536
+      # @param len [Integer] match length, at least 4
+      # @return [String] +out+
       def emit_copy(out, offset, len)
         while len >= 68
           emit_copy_upto64(out, offset, 64)
@@ -360,6 +476,12 @@ module Herringbone
         emit_copy_upto64(out, offset, len)
       end
 
+      # Appends one copy element: the 2-byte form (1-byte offset) when len is 4..11 and offset < 2048,
+      # otherwise the 3-byte form with a 2-byte offset.
+      # @param out [String] binary output buffer, appended to
+      # @param offset [Integer] backward distance to the match source, 1...65536
+      # @param len [Integer] copy length, 4..64
+      # @return [String] +out+
       def emit_copy_upto64(out, offset, len)
         if len < 12 && offset < 2048
           out << (1 | ((len - 4) << 2) | ((offset >> 8) << 5)) << (offset & 0xff)
@@ -368,7 +490,7 @@ module Herringbone
         end
       end
 
-      private_class_method :decompress_io_buffer, :decompress_string, :read_varint, :write_varint, :compress_block, :block_words, :match_length_words,
+      private_class_method :native, :native_library, :decompress_io_buffer, :decompress_string, :read_varint, :write_varint, :compress_block, :block_words, :match_length_words,
         :match_length, :emit_literal, :emit_copy, :emit_copy_upto64
     end
   end

data/lib/herringbone/compression.rb

@@ -4,62 +4,71 @@ require "zlib"
 require "stringio"
 
 module Herringbone
-  # Raised when a file uses (or a writer asks for) a codec whose library is not installed
+  # Raised when a file uses (or a writer asks for) a codec whose library is not loaded
   class MissingCodecError < UnsupportedError
+    # @return [String] codec name (+"ZSTD"+) and name of the gem providing it (+"zstd-ruby"+)
     attr_reader :codec, :gem_name
 
-    def initialize(codec, gem_name, load_error)
+    # @param codec [String] codec name, as shown in the message
+    # @param gem_name [String] gem to add to the Gemfile
+    # @param path [String] what to require to load the gem
+    def initialize(codec, gem_name, path)
       @codec = codec
       @gem_name = gem_name
-      super("#{codec} compression needs the \"#{gem_name}\" gem, which could not be loaded " \
-        "(#{load_error.message}). Add `gem \"#{gem_name}\"` to your Gemfile to use #{codec}.")
+      super("#{codec} compression needs the \"#{gem_name}\" gem, which is not loaded. Add " \
+        "`gem \"#{gem_name}\"` to your Gemfile and `require \"#{path}\"` to use #{codec}.")
     end
   end
 
   # Dispatches page (de)compression by Parquet codec id. Snappy and LZ4 are pure Ruby and GZIP
-  # uses zlib, so those always work. ZSTD and Brotli come from optional gems (zstd-ruby, brotli),
-  # which are required on first use; if they are missing, MissingCodecError says what to add.
+  # uses zlib, so those always work. LZO is pure Ruby too, but can only be read. ZSTD and Brotli
+  # come from optional gems (zstd-ruby, brotli), which the application requires; if they are
+  # not loaded, MissingCodecError says what to add.
+  # (Snappy also uses the optional snappy gem when it is loaded, see Codecs::Snappy.)
   module Compression
     module_function
 
     # Codecs backed by optional native gems: codec id => [name, gem, require path, constant]
-    LIBRARIES = {
+    LIBRARIES = Ractor.make_shareable({
       Format::Codec::ZSTD => ["ZSTD", "zstd-ruby", "zstd-ruby", :Zstd],
       Format::Codec::BROTLI => ["Brotli", "brotli", "brotli", :Brotli]
-    }.freeze
-
-    @libraries = {}
-    @library_mutex = Mutex.new
+    })
 
-    # The library module for a codec backed by an optional gem, requiring it on first use
+    # The library module for a codec backed by an optional gem
+    #
+    # @param codec [Integer] a codec id that is a key of LIBRARIES
+    # @return [Module] the gem's module (+Zstd+ or +Brotli+)
+    # @raise [MissingCodecError] when the gem is not loaded
+    # @raise [KeyError] for a codec id not in LIBRARIES
     def library(codec)
-      @libraries.fetch(codec) do
-        @library_mutex.synchronize do
-          @libraries.fetch(codec) do
-            name, gem_name, path, const = LIBRARIES.fetch(codec)
-            begin
-              require_library(path)
-            rescue LoadError => e
-              raise MissingCodecError.new(name, gem_name, e)
-            end
-            @libraries[codec] = Object.const_get(const)
-          end
-        end
-      end
+      name, gem_name, path, const = LIBRARIES.fetch(codec)
+      loaded_library(const) || raise(MissingCodecError.new(name, gem_name, path))
     end
 
-    def require_library(path)
-      require path
+    # A separate method so tests can stub it to simulate a missing gem
+    #
+    # @param const [Symbol] top-level constant the gem defines
+    # @return [Module, nil] the gem's module, nil when it is not loaded
+    def loaded_library(const)
+      Object.const_get(const) if Object.const_defined?(const)
     end
 
     # Raises MissingCodecError (or UnsupportedError) unless +codec+ can be used
+    #
+    # @param codec [Integer, Symbol, String] codec id or name (see NAMES)
+    # @return [Module, nil] the gem's module for a gem-backed codec, nil for a built-in one
+    # @raise [MissingCodecError] when the codec's gem is not loaded
+    # @raise [UnsupportedError] for a codec herringbone cannot write (LZO)
+    # @raise [ArgumentError] for an unknown codec name
     def ensure_available!(codec)
       codec = codec_id(codec)
       return library(codec) if LIBRARIES.key?(codec)
       return if SUPPORTED.include?(codec)
+      raise UnsupportedError, "LZO compression is only supported for reading" if codec == Format::Codec::LZO
       raise UnsupportedError, "#{Format::Codec::NAMES[codec] || codec} compression is not supported"
     end
 
+    # Codec ids that work without optional gems
     SUPPORTED = [
       Format::Codec::UNCOMPRESSED, Format::Codec::SNAPPY, Format::Codec::GZIP,
       Format::Codec::LZ4_RAW, Format::Codec::LZ4
@@ -71,8 +80,14 @@ module Herringbone
       Format::Codec::LZ4_RAW => :lz4, Format::Codec::LZ4 => :lz4_hadoop, Format::Codec::ZSTD => :zstd,
       Format::Codec::BROTLI => :brotli, Format::Codec::LZO => :lzo
     }.freeze
+    # Codec name => codec id, the inverse of NAMES
     CODECS_BY_NAME = NAMES.invert.freeze
 
+    # Codec id for a codec name; Integers are taken to be ids already and returned unchecked
+    #
+    # @param name [Integer, Symbol, String] codec id, or a name from NAMES (case-insensitive)
+    # @return [Integer] the Format::Codec id
+    # @raise [ArgumentError] for an unknown name
     def codec_id(name)
       return name if name.is_a?(Integer)
       CODECS_BY_NAME.fetch(name.to_s.downcase.to_sym) do
@@ -80,6 +95,15 @@ module Herringbone
       end
     end
 
+    # Decompresses a page body, checking the result against the size the page header declares
+    #
+    # @param codec [Integer] Format::Codec id from the column chunk metadata
+    # @param data [String] compressed bytes
+    # @param uncompressed_size [Integer] expected decompressed size in bytes
+    # @return [String] decompressed bytes (binary)
+    # @raise [UnsupportedError] for a codec herringbone does not implement
+    # @raise [MissingCodecError] when the codec's gem is not loaded
+    # @raise [FormatError] when the decompressed size does not match +uncompressed_size+
     def decompress(codec, data, uncompressed_size)
       return "".b if uncompressed_size.zero? && data.empty?
       out = case codec
@@ -88,6 +112,7 @@ module Herringbone
       when Format::Codec::GZIP then gunzip(data)
       when Format::Codec::LZ4_RAW then Codecs::LZ4.decompress_block(data, uncompressed_size)
       when Format::Codec::LZ4 then Codecs::LZ4.decompress_hadoop(data, uncompressed_size)
+      when Format::Codec::LZO then Codecs::LZO.decompress_hadoop(data, uncompressed_size)
       when Format::Codec::ZSTD then library(codec).decompress(data)
       when Format::Codec::BROTLI then library(codec).inflate(data)
       else
@@ -100,6 +125,13 @@ module Herringbone
       out
     end
 
+    # Compresses a page body
+    #
+    # @param codec [Integer] Format::Codec id
+    # @param data [String] bytes to compress
+    # @return [String] compressed bytes (binary)
+    # @raise [UnsupportedError] for a codec herringbone does not implement
+    # @raise [MissingCodecError] when the codec's gem is not loaded
     def compress(codec, data)
       case codec
       when Format::Codec::UNCOMPRESSED then data
@@ -115,6 +147,10 @@ module Herringbone
     end
 
     # Handles files whose gzip data consists of several concatenated members
+    #
+    # @param data [String] gzip data, one or more members
+    # @return [String] the members' decompressed bytes concatenated (binary)
+    # @raise [Zlib::GzipFile::Error] on corrupt gzip data
     def gunzip(data)
       io = StringIO.new(data)
       out = String.new(encoding: Encoding::BINARY)