herringbone 0.3.0 → 0.5.0

data/lib/herringbone/codecs/lzo.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module Herringbone
+  module Codecs
+    # Pure-Ruby LZO1X decompressor, for reading Parquet files written by parquet-mr with
+    # hadoop-lzo. There is no compressor: nothing but hadoop-lzo writes LZO, and nothing
+    # outside the JVM reads it.
+    #
+    # Written from the bitstream description in the Linux kernel's
+    # Documentation/staging/lzo.rst, not from the (GPL) LZO sources.
+    module LZO
+      # Raised for corrupt or truncated LZO input.
+      class Error < StandardError; end
+
+      # Hadoop block header: big-endian 32-bit uncompressed size of the block.
+      HADOOP_BLOCK_PREFIX = 4
+      # Hadoop chunk header: big-endian 32-bit compressed size of the chunk.
+      HADOOP_CHUNK_PREFIX = 4
+      # Shorthand for Encoding::BINARY.
+      BINARY = Encoding::BINARY
+
+      module_function
+
+      # Parquet LZO (codec 3). hadoop-lzo writes Hadoop block framing; data that does not fit
+      # the framing is decoded as one bare LZO1X stream.
+      # @param input [String] compressed page data
+      # @param uncompressed_size [Integer] exact decompressed size, from the page header
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if neither layout decodes
+      def decompress_hadoop(input, uncompressed_size)
+        src = binary(input)
+        try_hadoop(src, uncompressed_size) || decompress_block(src, uncompressed_size)
+      end
+
+      # Decompress one bare LZO1X stream that must expand to exactly uncompressed_size bytes.
+      # @param input [String] LZO1X stream, ending with the end-of-stream marker
+      # @param uncompressed_size [Integer] exact decompressed size
+      # @return [String] decompressed bytes in ASCII-8BIT
+      # @raise [Error] if the stream is corrupt or does not decode to +uncompressed_size+ bytes
+      def decompress_block(input, uncompressed_size)
+        src = binary(input)
+        out = String.new(capacity: uncompressed_size, encoding: BINARY)
+        ip = decode_block(src, 0, src.bytesize, out, uncompressed_size)
+        raise Error, "#{src.bytesize - ip} bytes after the end of the LZO stream" unless ip == src.bytesize
+        unless out.bytesize == uncompressed_size
+          raise Error, "LZO stream decoded to #{out.bytesize} bytes, expected #{uncompressed_size}"
+        end
+        out
+      end
+
+      # @param str [String] input in any encoding
+      # @return [String] +str+ itself if already binary, otherwise a binary copy
+      def binary(str)
+        (str.encoding == BINARY) ? str : str.b
+      end
+
+      # Hadoop's BlockCompressorStream layout: blocks of (uncompressed size, then chunks of
+      # (compressed size, LZO stream) until the block is filled). Each chunk is a separate stream.
+      # @param src [String] binary input
+      # @param uncompressed_size [Integer] exact decompressed size expected over all blocks
+      # @return [String, nil] decompressed bytes, or nil if +src+ is not valid Hadoop-framed LZO
+      def try_hadoop(src, uncompressed_size)
+        n = src.bytesize
+        out = String.new(capacity: uncompressed_size, encoding: BINARY)
+        ip = 0
+        while ip < n
+          return nil if n - ip < HADOOP_BLOCK_PREFIX
+          block_end = out.bytesize + src.byteslice(ip, HADOOP_BLOCK_PREFIX).unpack1("N")
+          ip += HADOOP_BLOCK_PREFIX
+          return nil if block_end > uncompressed_size
+
+          while out.bytesize < block_end
+            return nil if n - ip < HADOOP_CHUNK_PREFIX
+            csize = src.byteslice(ip, HADOOP_CHUNK_PREFIX).unpack1("N")
+            ip += HADOOP_CHUNK_PREFIX
+            return nil if csize > n - ip
+            begin
+              return nil unless decode_block(src, ip, ip + csize, out, block_end) == ip + csize
+            rescue Error
+              return nil
+            end
+            ip += csize
+          end
+        end
+        (out.bytesize == uncompressed_size) ? out : nil
+      end
+
+      # Decode one LZO1X stream from src[ip...iend], appending to out. Matches may only reach
+      # back to where this stream's output starts. out may not grow beyond limit.
+      #
+      # Every instruction is a match optionally followed by 0-3 literals (its low two bits, the
+      # "state"), or a run of 4+ literals. What an instruction byte below 16 means depends on
+      # the state the previous instruction left behind.
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the stream in +src+
+      # @param iend [Integer] offset just past the stream
+      # @param out [String] binary output buffer, appended to
+      # @param limit [Integer] maximum total byte size of +out+
+      # @return [Integer] input offset just past the end-of-stream marker
+      # @raise [Error] if the stream is truncated, has an invalid distance or overflows +limit+
+      def decode_block(src, ip, iend, out, limit)
+        base = out.bytesize
+        raise Error, "Empty LZO stream" if ip >= iend
+
+        state = 0
+        t = src.getbyte(ip)
+        if t > 17 # the first byte may open with a literal run of its own
+          ip += 1
+          state = t - 17
+          ip = copy_literals(src, ip, iend, out, limit, state)
+          state = 4 if state > 4
+        end
+
+        while true
+          raise Error, "Truncated LZO stream" if ip >= iend
+          t = src.getbyte(ip)
+          ip += 1
+
+          if t < 16
+            if state == 0
+              len = t + 3
+              len, ip = extended_length(src, ip, iend, 18) if t == 0
+              ip = copy_literals(src, ip, iend, out, limit, len)
+              state = 4
+              next
+            end
+            raise Error, "Truncated LZO stream" if ip >= iend
+            dist = (src.getbyte(ip) << 2) + (t >> 2) + 1
+            ip += 1
+            if state == 4
+              dist += 2048
+              len = 3
+            else
+              len = 2
+            end
+            s = t & 3
+          elsif t < 32
+            len = (t & 7) + 2
+            len, ip = extended_length(src, ip, iend, 9) if t & 7 == 0
+            raise Error, "Truncated LZO stream" if ip + 2 > iend
+            le = src.getbyte(ip) | (src.getbyte(ip + 1) << 8)
+            ip += 2
+            dist = ((t & 8) << 11) + (le >> 2)
+            return ip if dist == 0 # end of stream
+            dist += 16_384
+            s = le & 3
+          elsif t < 64
+            len = (t & 31) + 2
+            len, ip = extended_length(src, ip, iend, 33) if t & 31 == 0
+            raise Error, "Truncated LZO stream" if ip + 2 > iend
+            le = src.getbyte(ip) | (src.getbyte(ip + 1) << 8)
+            ip += 2
+            dist = (le >> 2) + 1
+            s = le & 3
+          else
+            raise Error, "Truncated LZO stream" if ip >= iend
+            len = (t >> 5) + 1 # 3-4 bytes for 01LDDDSS, 5-8 for 1LLDDDSS
+            dist = (src.getbyte(ip) << 3) + ((t >> 2) & 7) + 1
+            ip += 1
+            s = t & 3
+          end
+
+          copy_match(out, base, dist, len, limit)
+          ip = copy_literals(src, ip, iend, out, limit, s) if s > 0
+          state = s
+        end
+      end
+
+      # Reads a length extension: a byte of 0 for each 255, then the nonzero remainder.
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the first extension byte
+      # @param iend [Integer] offset just past the stream
+      # @param bias [Integer] added to the extension
+      # @return [Array(Integer, Integer)] the length, and the offset just past the extension
+      # @raise [Error] if the extension runs past +iend+
+      def extended_length(src, ip, iend, bias)
+        len = bias
+        while true
+          raise Error, "Truncated LZO length" if ip >= iend
+          b = src.getbyte(ip)
+          ip += 1
+          return [len + b, ip] unless b == 0
+          len += 255
+        end
+      end
+
+      # @param src [String] binary input
+      # @param ip [Integer] offset of the literals in +src+
+      # @param iend [Integer] offset just past the stream
+      # @param out [String] binary output buffer, appended to
+      # @param limit [Integer] maximum total byte size of +out+
+      # @param len [Integer] number of literal bytes
+      # @return [Integer] offset just past the literals
+      # @raise [Error] if the literals run past +iend+ or +out+ would exceed +limit+
+      def copy_literals(src, ip, iend, out, limit, len)
+        raise Error, "LZO literals run past end of input" if ip + len > iend
+        raise Error, "LZO output exceeds expected size" if out.bytesize + len > limit
+        out << src.byteslice(ip, len)
+        ip + len
+      end
+
+      # @param out [String] binary output buffer, appended to
+      # @param base [Integer] size of +out+ when the current stream started
+      # @param dist [Integer] how far back the match starts
+      # @param len [Integer] match length, which may exceed +dist+ (a repeating pattern)
+      # @param limit [Integer] maximum total byte size of +out+
+      # @return [String] +out+
+      # @raise [Error] if the match reaches before the stream's output or +out+ would exceed +limit+
+      def copy_match(out, base, dist, len, limit)
+        pos = out.bytesize
+        raise Error, "LZO match distance #{dist} reaches before start of output" if dist > pos - base
+        raise Error, "LZO output exceeds expected size" if pos + len > limit
+        if dist >= len
+          out << out.byteslice(pos - dist, len)
+        else
+          pattern = out.byteslice(pos - dist, dist)
+          out << (pattern * (len / dist)) << pattern.byteslice(0, len % dist)
+        end
+      end
+
+      private_class_method :binary, :try_hadoop, :decode_block, :extended_length, :copy_literals, :copy_match
+    end
+  end
+end

data/lib/herringbone/codecs/snappy.rb

@@ -7,7 +7,7 @@ module Herringbone
     # Pure-Ruby implementation of the raw Snappy block format (as used by Parquet),
     # see https://github.com/google/snappy/blob/main/format_description.txt
     #
-    # When the `snappy` gem (a binding to Google's libsnappy) can be loaded, it is used instead:
+    # When the `snappy` gem (a binding to Google's libsnappy) is loaded, it is used instead:
     # 12x faster decompression and 27x faster compression. It is optional and only a speedup: if
     # it is missing, the pure-Ruby code below is used silently. Both produce raw Snappy blocks the
     # other reads.
@@ -30,7 +30,7 @@ module Herringbone
       # The length preamble is a 32-bit varint, so no block may decompress to more than this.
       MAX_UNCOMPRESSED = (1 << 32) - 1
 
-      # Name of the optional native gem, passed to +require+.
+      # Name of the optional native gem.
       NATIVE_GEM = "snappy"
 
       module_function
@@ -57,41 +57,34 @@ module Herringbone
         native ? :native : :ruby
       end
 
-      # For tests and benchmarks: :ruby forces pure Ruby, :native requires the snappy gem
-      # (UnsupportedError if it cannot be loaded), nil goes back to the default
+      # For tests and benchmarks: :ruby forces pure Ruby, :native the snappy gem
+      # (UnsupportedError if it is not loaded), nil goes back to the default
       # @param name [Symbol, nil] +:ruby+, +:native+ or nil
       # @return [void]
-      # @raise [UnsupportedError] if +:native+ is requested and the gem cannot be loaded
+      # @raise [UnsupportedError] if +:native+ is requested and the gem is not loaded
       # @raise [ArgumentError] for any other backend name
       def backend=(name)
         @native = case name
         when :ruby then false
         when :native
-          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem could not be loaded")
+          native_library || raise(UnsupportedError, "The \"#{NATIVE_GEM}\" gem is not loaded (require \"#{NATIVE_GEM}\")")
         when nil then nil
         else raise ArgumentError, "Unknown Snappy backend #{name.inspect} (expected :ruby, :native or nil)"
         end
       end
 
-      # The native library to use, resolved (and memoized) on first call unless forced by backend=.
+      # The native library to use: the snappy gem when it is loaded, unless backend= forced one.
       # @return [Module, nil] the +::Snappy+ module, or nil when the pure-Ruby code should run
       def native
-        @native = native_library || false if @native.nil?
-        @native || nil
+        native = @native
+        native = native_library if native.nil?
+        native || nil
       end
 
-      # Tries to load the snappy gem once and memoizes the result.
-      # @return [Module, false] the +::Snappy+ module, or false if it is missing or lacks inflate/deflate
+      # Not memoized, so that it works the same in every Ractor and once the gem is required later.
+      # @return [Module, false] the +::Snappy+ module, or false if it is not loaded or lacks inflate/deflate
       def native_library
-        if @native_lib.nil?
-          @native_lib = begin
-            require NATIVE_GEM
-            (::Snappy.respond_to?(:inflate) && ::Snappy.respond_to?(:deflate)) ? ::Snappy : false
-          rescue LoadError
-            false
-          end
-        end
-        @native_lib
+        (defined?(::Snappy.inflate) && defined?(::Snappy.deflate)) ? ::Snappy : false
       end
 
       # Decompresses into a preallocated IO::Buffer: copies do not allocate intermediate Strings.

data/lib/herringbone/compression.rb

@@ -4,80 +4,67 @@ require "zlib"
 require "stringio"
 
 module Herringbone
-  # Raised when a file uses (or a writer asks for) a codec whose library is not installed
+  # Raised when a file uses (or a writer asks for) a codec whose library is not loaded
   class MissingCodecError < UnsupportedError
     # @return [String] codec name (+"ZSTD"+) and name of the gem providing it (+"zstd-ruby"+)
     attr_reader :codec, :gem_name
 
     # @param codec [String] codec name, as shown in the message
     # @param gem_name [String] gem to add to the Gemfile
-    # @param load_error [LoadError] the error from requiring the gem, whose message is included
-    def initialize(codec, gem_name, load_error)
+    # @param path [String] what to require to load the gem
+    def initialize(codec, gem_name, path)
       @codec = codec
       @gem_name = gem_name
-      super("#{codec} compression needs the \"#{gem_name}\" gem, which could not be loaded " \
-        "(#{load_error.message}). Add `gem \"#{gem_name}\"` to your Gemfile to use #{codec}.")
+      super("#{codec} compression needs the \"#{gem_name}\" gem, which is not loaded. Add " \
+        "`gem \"#{gem_name}\"` to your Gemfile and `require \"#{path}\"` to use #{codec}.")
     end
   end
 
   # Dispatches page (de)compression by Parquet codec id. Snappy and LZ4 are pure Ruby and GZIP
-  # uses zlib, so those always work. ZSTD and Brotli come from optional gems (zstd-ruby, brotli),
-  # which are required on first use; if they are missing, MissingCodecError says what to add.
-  # (Snappy also uses the optional snappy gem when it is installed, see Codecs::Snappy.)
+  # uses zlib, so those always work. LZO is pure Ruby too, but can only be read. ZSTD and Brotli
+  # come from optional gems (zstd-ruby, brotli), which the application requires; if they are
+  # not loaded, MissingCodecError says what to add.
+  # (Snappy also uses the optional snappy gem when it is loaded, see Codecs::Snappy.)
   module Compression
     module_function
 
     # Codecs backed by optional native gems: codec id => [name, gem, require path, constant]
-    LIBRARIES = {
+    LIBRARIES = Ractor.make_shareable({
       Format::Codec::ZSTD => ["ZSTD", "zstd-ruby", "zstd-ruby", :Zstd],
       Format::Codec::BROTLI => ["Brotli", "brotli", "brotli", :Brotli]
-    }.freeze
-
-    @libraries = {}
-    @library_mutex = Mutex.new
+    })
 
-    # The library module for a codec backed by an optional gem, requiring it on first use
+    # The library module for a codec backed by an optional gem
     #
     # @param codec [Integer] a codec id that is a key of LIBRARIES
     # @return [Module] the gem's module (+Zstd+ or +Brotli+)
-    # @raise [MissingCodecError] when the gem cannot be loaded
+    # @raise [MissingCodecError] when the gem is not loaded
     # @raise [KeyError] for a codec id not in LIBRARIES
     def library(codec)
-      @libraries.fetch(codec) do
-        @library_mutex.synchronize do
-          @libraries.fetch(codec) do
-            name, gem_name, path, const = LIBRARIES.fetch(codec)
-            begin
-              require_library(path)
-            rescue LoadError => e
-              raise MissingCodecError.new(name, gem_name, e)
-            end
-            @libraries[codec] = Object.const_get(const)
-          end
-        end
-      end
+      name, gem_name, path, const = LIBRARIES.fetch(codec)
+      loaded_library(const) || raise(MissingCodecError.new(name, gem_name, path))
     end
 
-    # Requires a codec gem; a separate method so tests can stub it to simulate a missing gem
+    # A separate method so tests can stub it to simulate a missing gem
     #
-    # @param path [String] require path of the gem
-    # @return [Boolean] whether the library was newly loaded
-    # @raise [LoadError] when the gem is not installed
-    def require_library(path)
-      require path
+    # @param const [Symbol] top-level constant the gem defines
+    # @return [Module, nil] the gem's module, nil when it is not loaded
+    def loaded_library(const)
+      Object.const_get(const) if Object.const_defined?(const)
     end
 
     # Raises MissingCodecError (or UnsupportedError) unless +codec+ can be used
     #
     # @param codec [Integer, Symbol, String] codec id or name (see NAMES)
     # @return [Module, nil] the gem's module for a gem-backed codec, nil for a built-in one
-    # @raise [MissingCodecError] when the codec's gem cannot be loaded
-    # @raise [UnsupportedError] for a codec herringbone does not implement (LZO)
+    # @raise [MissingCodecError] when the codec's gem is not loaded
+    # @raise [UnsupportedError] for a codec herringbone cannot write (LZO)
     # @raise [ArgumentError] for an unknown codec name
     def ensure_available!(codec)
       codec = codec_id(codec)
       return library(codec) if LIBRARIES.key?(codec)
       return if SUPPORTED.include?(codec)
+      raise UnsupportedError, "LZO compression is only supported for reading" if codec == Format::Codec::LZO
       raise UnsupportedError, "#{Format::Codec::NAMES[codec] || codec} compression is not supported"
     end
 
@@ -115,7 +102,7 @@ module Herringbone
     # @param uncompressed_size [Integer] expected decompressed size in bytes
     # @return [String] decompressed bytes (binary)
     # @raise [UnsupportedError] for a codec herringbone does not implement
-    # @raise [MissingCodecError] when the codec's gem cannot be loaded
+    # @raise [MissingCodecError] when the codec's gem is not loaded
     # @raise [FormatError] when the decompressed size does not match +uncompressed_size+
     def decompress(codec, data, uncompressed_size)
       return "".b if uncompressed_size.zero? && data.empty?
@@ -125,6 +112,7 @@ module Herringbone
       when Format::Codec::GZIP then gunzip(data)
       when Format::Codec::LZ4_RAW then Codecs::LZ4.decompress_block(data, uncompressed_size)
       when Format::Codec::LZ4 then Codecs::LZ4.decompress_hadoop(data, uncompressed_size)
+      when Format::Codec::LZO then Codecs::LZO.decompress_hadoop(data, uncompressed_size)
       when Format::Codec::ZSTD then library(codec).decompress(data)
       when Format::Codec::BROTLI then library(codec).inflate(data)
       else
@@ -143,7 +131,7 @@ module Herringbone
     # @param data [String] bytes to compress
     # @return [String] compressed bytes (binary)
     # @raise [UnsupportedError] for a codec herringbone does not implement
-    # @raise [MissingCodecError] when the codec's gem cannot be loaded
+    # @raise [MissingCodecError] when the codec's gem is not loaded
     def compress(codec, data)
       case codec
       when Format::Codec::UNCOMPRESSED then data

data/lib/herringbone/encodings/plain.rb

@@ -8,12 +8,12 @@ module Herringbone
       module_function
 
       # Fixed-width numeric types: pack/unpack directive and byte width (all little-endian).
-      FORMATS = {
+      FORMATS = Ractor.make_shareable({
         Format::Type::INT32 => ["l<", 4],
         Format::Type::INT64 => ["q<", 8],
         Format::Type::FLOAT => ["e", 4],
         Format::Type::DOUBLE => ["E", 8]
-      }.freeze
+      })
 
       # Decodes +count+ values of +type+ from +data+ starting at +pos+.
       # Returns [values, new_pos].

data/lib/herringbone/inferring_writer.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module Herringbone
+  # Writes rows whose schema is inferred from the rows themselves, reading them only once: the
+  # first Schema::INFER_SAMPLE rows are held back, the schema is built from them, and then they and
+  # every later row go straight to a Writer. Memory is bounded by the sample, however many rows
+  # follow it. Nothing is written to the IO before the schema is known, so a source that can only
+  # be iterated once (a cursor, a queue, a lazy Enumerator over an IO) loses no rows.
+  #
+  # A row that does not fit the inferred schema raises SchemaMismatch and stops the writer: the
+  # file is left unfinished (no footer) rather than written with rows missing.
+  #
+  # Used by Herringbone.write and SimpleWriter.
+  #
+  # @api private
+  class InferringWriter
+    # @param io [IO, #write] destination, passed to Writer.new
+    # @param fix [String] how a caller declares a column, for SchemaMismatch messages; +%s+ is
+    #   replaced by a declaration such as +string :age+
+    # @param options [Hash{Symbol => Object}] Writer options
+    # @option options [Symbol] :compression (:snappy) codec, checked before any row is read
+    #   (the other Writer options are passed on as well)
+    # @yield [sample] builds the schema once the sample is complete (or at #close)
+    # @yieldparam sample [Array] the rows held back, at most Schema::INFER_SAMPLE
+    # @yieldreturn [Schema]
+    # @raise [MissingCodecError] when the codec's optional gem is not loaded
+    def initialize(io, fix:, **options, &schema_for)
+      # Fail before reading any rows if the codec's library is missing
+      Compression.ensure_available!(Compression.codec_id(options.fetch(:compression, :snappy)))
+      @io = io
+      @fix = fix
+      @options = options
+      @schema_for = schema_for
+      @sample = []
+      @sample_size = nil
+      @writer = nil
+      @mismatch = nil
+    end
+
+    # @param row [Object] row to write, in any form the Writer accepts
+    # @return [self]
+    # @raise [SchemaMismatch] when the row does not fit the inferred schema
+    # @raise [Error] after an earlier SchemaMismatch
+    def <<(row)
+      check_usable!
+      if @writer
+        write(row)
+      else
+        @sample << row
+        start if @sample.size >= Schema::INFER_SAMPLE
+      end
+      self
+    end
+
+    # @return [Integer] rows accepted so far, including the ones held back
+    def rows_written = @writer ? @writer.rows_written : @sample.size
+
+    # Writes the held-back rows if the sample never filled up, then finishes the file
+    #
+    # @return [void]
+    # @raise [SchemaMismatch] when a held-back row does not fit the inferred schema
+    # @raise [Error] after an earlier SchemaMismatch
+    def close
+      check_usable!
+      start unless @writer
+      @writer.close
+    end
+
+    # Stops without finishing the file; when the schema was never built nothing was written at all
+    #
+    # @return [void]
+    def abort
+      @writer&.abort
+    end
+
+    # Short summary for the console, without the held-back rows
+    #
+    # @return [String] the number of rows held back, or the Writer's summary once it is open
+    def inspect
+      "#<#{self.class.name} #{@writer ? "writer=#{@writer.inspect}" : "held_back=#{@sample.size}"}>"
+    end
+
+    private
+
+    # Builds the schema from the held-back rows, opens the Writer and writes them
+    #
+    # @return [void]
+    # @raise [SchemaMismatch] when a held-back row does not fit
+    def start
+      @sample_size = @sample.size
+      @writer = Writer.new(@io, @schema_for.call(@sample), **@options)
+      sample, @sample = @sample, nil
+      sample.each { |row| write(row) }
+    end
+
+    # @param row [Object] row for the Writer
+    # @return [void]
+    # @raise [SchemaMismatch] when the row does not fit; the Writer is aborted
+    def write(row)
+      @writer << row
+    rescue EncodeError => e
+      @writer.abort
+      @mismatch = SchemaMismatch.new(explain(e), row: e.row, column: e.column, value: e.value)
+      raise @mismatch
+    end
+
+    # @return [void]
+    # @raise [Error] after a SchemaMismatch
+    def check_usable!
+      return unless @mismatch
+      raise Error, "This writer stopped at a schema mismatch and cannot continue " \
+        "(the file is unfinished):\n#{@mismatch.message}"
+    end
+
+    # A multi-line explanation of an EncodeError against the inferred schema
+    #
+    # @param error [EncodeError] error raised by the Writer
+    # @return [String]
+    def explain(error)
+      column = error.column && @writer.schema.column(error.column)
+      column ||= error.column && @writer.schema.columns.find { |c| c.dotted_path.start_with?("#{error.column}.") }
+      top = error.column&.split(".")&.first
+      lines = ["Row #{error.row} does not fit the schema inferred from the first #{@sample_size} rows."]
+      lines << "  column:   #{error.column}" if error.column
+      lines << "  inferred: #{describe(column)}" if column
+      lines << "  got:      #{error.value.inspect[0, 200]} (#{error.value.class})" unless error.value.nil?
+      lines << "  error:    #{error.message.sub(/\ARow \d+: /, "")}"
+      lines << ""
+      lines << "Column types are decided from the first #{Schema::INFER_SAMPLE} rows (or all of them, when there are"
+      lines << "fewer), so a later value of a different type cannot be written to the same column."
+      if top
+        lines << "Declare the column with a type that holds every value, for example:"
+        lines << "  #{format(@fix, declaration_for(top, error.value))}"
+      end
+      lines << "or pass a complete schema. The file was left unfinished (no footer)."
+      lines.join("\n")
+    end
+
+    # @param column [Schema::Column] leaf column
+    # @return [String] its type as the Builder DSL names it, e.g. "int64" or "timestamp (micros)"
+    def describe(column)
+      kind, *details = Types.logical_of(column.node)
+      type = case kind
+      when nil then Format::Type::NAMES[column.type].downcase
+      when :integer then "#{details[1] ? "int" : "uint"}#{details[0]}"
+      when :timestamp, :time then "#{kind} (#{details[0]})"
+      when :decimal then "decimal(#{details[1]}, #{details[0]})"
+      else kind.to_s
+      end
+      (column.path.size > 1) ? "#{type} (at #{column.dotted_path})" : type
+    end
+
+    # A Builder DSL declaration whose type can hold +value+, to suggest in the explanation
+    #
+    # @param name [String] top-level field name
+    # @param value [Object] the value that did not fit
+    # @return [String] e.g. "string :age"
+    def declaration_for(name, value)
+      type = case value
+      when Float, BigDecimal, Rational then "double"
+      when Time, DateTime then "timestamp"
+      when Hash, Array then "json"
+      else "string"
+      end
+      "#{type} :#{name}"
+    end
+  end
+end

data/lib/herringbone/reader/column_chunk_reader.rb

@@ -292,8 +292,8 @@ module Herringbone
         rep_len = dh.repetition_levels_byte_length
         def_len = dh.definition_levels_byte_length
         reps = defs = nil
-        reps = PageStream::HybridDecoder.new(body, 0, rep_len, RLE_WIDTH[@max_rep]) if @max_rep.positive?
-        defs = PageStream::HybridDecoder.new(body, rep_len, rep_len + def_len, RLE_WIDTH[@max_def]) if @max_def.positive?
+        reps = PageStream::HybridDecoder.new(body, 0, rep_len, @max_rep.bit_length) if @max_rep.positive?
+        defs = PageStream::HybridDecoder.new(body, rep_len, rep_len + def_len, @max_def.bit_length) if @max_def.positive?
         data = body.byteslice(rep_len + def_len, body.bytesize - rep_len - def_len)
         if dh.is_compressed != false
           data = decompress(data, header.uncompressed_page_size - rep_len - def_len)
@@ -302,9 +302,6 @@ module Herringbone
         PageStream::Page.new(n, defs, reps, values, conv)
       end
 
-      # Bit width of levels up to a max level (memoized Integer#bit_length)
-      RLE_WIDTH = Hash.new { |h, k| h[k] = k.bit_length }
-
       # [decoder, position after the levels]
       #
       # @param data [String] the uncompressed page
@@ -317,7 +314,7 @@ module Herringbone
       # @raise [FormatError] when the RLE length prefix is cut off
       # @raise [UnsupportedError] for any other level encoding
       def level_decoder(data, pos, encoding, max, n)
-        width = RLE_WIDTH[max]
+        width = max.bit_length
         case encoding
         when E::RLE
           len = data.byteslice(pos, 4)&.unpack1("V") or raise FormatError, "Truncated levels"
@@ -332,9 +329,9 @@ module Herringbone
       end
 
       # Physical type => [String#unpack directive, byte width] for PLAIN fixed-width values
-      FIXED_FORMATS = {
+      FIXED_FORMATS = Ractor.make_shareable({
         T::INT32 => ["l<", 4], T::INT64 => ["q<", 8], T::FLOAT => ["e", 4], T::DOUBLE => ["E", 8]
-      }.freeze
+      })
 
       # [value decoder, converter still to apply to its values (nil for dictionary pages)]
       #