herringbone 0.2.0 → 0.3.0

data/lib/herringbone/io_buffer_support.rb

@@ -5,6 +5,8 @@ module Herringbone
   # It still carries an "experimental" warning, which is silenced once here: the gem only uses
   # new/for/copy/get_string/free, and falls back to String operations where it is missing.
   module IOBufferSupport
+    # true when IO::Buffer has the methods the decompressors use and HERRINGBONE_NO_IO_BUFFER is
+    # not set in the environment (which forces the String fallback)
     AVAILABLE = begin
       if defined?(IO::Buffer) && IO::Buffer.method_defined?(:copy) && IO::Buffer.method_defined?(:get_string)
         previous = Warning[:experimental]
@@ -18,7 +20,7 @@ module Herringbone
       else
         false
       end
-    rescue StandardError
+    rescue
       false
     end
   end

data/lib/herringbone/reader/column_chunk_reader.rb

@@ -14,22 +14,35 @@ module Herringbone
     # The declared total_compressed_size of the chunk is not relied on (some old writers
     # under-report it): pages are read until the chunk's num_values have been seen.
     class ColumnChunkReader
+      # Shorthand for the physical type constants
       T = Format::Type
+      # Shorthand for the encoding constants
       E = Format::Encoding
 
       # Bytes read past a page body, so that the next page header usually needs no extra read
       READ_AHEAD = 64 * 1024
+      # Bytes read for a page header whose size is not known; grown 4x while decoding fails
       HEADER_GUESS = 1024
+      # Largest header read attempted before a page header is declared corrupt
       MAX_HEADER = 64 * 1024 * 1024
 
+      # @return [Schema::Column] the leaf column this chunk belongs to
       attr_reader :column
 
       # With +lazy+, values of data pages that are not dictionary-encoded are returned as
       # physical values, and #page_converter is what still has to be applied to them. This keeps
       # a decoded page small (Integers instead of Time or BigDecimal objects) when only a slice
       # of it is needed at a time.
+      #
+      # @return [Proc, nil] converter of the last page returned by #next_page, nil when none
       attr_reader :page_converter
 
+      # @param io [IO, StringIO] the file, read with #seek and #read
+      # @param chunk [Format::ColumnChunk] the chunk's footer entry
+      # @param column [Schema::Column] the leaf column the chunk stores
+      # @param converter [Proc, nil] physical value => Ruby value, applied to values and dictionaries
+      # @param lazy [Boolean] leave non-dictionary values physical in #next_page (see #page_converter)
+      # @raise [UnsupportedError] for chunks without metadata (encrypted) or stored in another file
       def initialize(io, chunk, column, converter: column.converter, lazy: false)
         @io = io
         @chunk = chunk
@@ -43,7 +56,7 @@ module Herringbone
         start = @meta.data_page_offset
         dict = @meta.dictionary_page_offset
         # Some writers store 0 when there is no dictionary page
-        start = dict if dict && dict.positive? && dict < start
+        start = dict if dict&.positive? && dict < start
         @pos = start
         @start = start
         @locations = nil # OffsetIndex page locations, when jumping between pages
@@ -55,6 +68,10 @@ module Herringbone
       end
 
       # All pages concatenated: [definition_levels, repetition_levels, values]
+      #
+      # @return [Array(Array<Integer>, Array<Integer>, Array)] levels are nil when the column's
+      #   max level is 0
+      # @raise [FormatError] when a page is corrupt or overruns the file
       def read
         defs = @max_def.positive? ? [] : nil
         reps = @max_rep.positive? ? [] : nil
@@ -69,6 +86,10 @@ module Herringbone
       end
 
       # The next data page as [defs, reps, values], or nil after the last one
+      #
+      # @return [Array(Array<Integer>, Array<Integer>, Array), nil] levels are nil when the
+      #   column's max level is 0; values hold the non-null entries
+      # @raise [FormatError] when a page is corrupt or overruns the file
       def next_page
         page = next_stream or return nil
         n = page.remaining
@@ -89,6 +110,11 @@ module Herringbone
 
       # The next data page as a PageStream::Page that decodes its levels and values on demand,
       # or nil after the last one. Values come out physical; apply Page#converter to them.
+      # Dictionary pages are read on the way.
+      #
+      # @return [PageStream::Page, nil] the next data page
+      # @raise [FormatError] when a page header is corrupt or a page overruns the file
+      # @raise [UnsupportedError] for an unsupported encoding or compression codec
       def next_stream
         while more_pages?
           header, body = read_page
@@ -112,11 +138,17 @@ module Herringbone
         raise FormatError, "Corrupt page header in #{@column.dotted_path}: #{e.message}"
       end
 
-      # The chunk's data page locations from its OffsetIndex (enables #jump_to_page)
+      # @return [Array<Format::PageLocation>, nil] the chunk's data page locations from its
+      #   OffsetIndex (enables #jump_to_page)
       attr_accessor :locations
 
       # Continues reading at data page +index+ of the OffsetIndex. The dictionary page (which
       # the OffsetIndex does not list) is read first if it has not been yet.
+      #
+      # @param index [Integer] position of the page in #locations
+      # @return [void]
+      # @raise [ArgumentError] when no #locations are set
+      # @raise [IndexError] when +index+ is outside #locations
       def jump_to_page(index)
         raise ArgumentError, "No OffsetIndex for #{@column.dotted_path}" unless @locations
         load_dictionary
@@ -125,18 +157,29 @@ module Herringbone
       end
 
       # Whether all of the chunk's values have been returned
+      #
+      # @return [Boolean] true once num_values entries have been seen
       def done? = @seen >= @total
 
-      def seen = @seen
-      def total = @total
+      # @return [Integer] entries (levels, nulls included) of the data pages returned so far
+      attr_reader :seen
+
+      # @return [Integer] the chunk's num_values from its ColumnMetaData
+      attr_reader :total
 
       private
 
+      # Whether another data page is due: up to the last OffsetIndex location after a jump,
+      # otherwise until num_values entries have been seen
+      #
+      # @return [Boolean] true when #next_stream should read on
       def more_pages?
         @page_number ? @page_number < @locations.size : @seen < @total
       end
 
       # Reads the dictionary page at the start of the chunk, if there is one
+      #
+      # @return [void]
       def load_dictionary
         return if @dictionary || @dictionary_checked
         @dictionary_checked = true
@@ -150,6 +193,9 @@ module Herringbone
       end
 
       # Reads the page header at @pos and the page body after it
+      #
+      # @return [Array(Format::PageHeader, String)] the header and the (still compressed) body
+      # @raise [FormatError] for a negative page size or a page that overruns the file
       def read_page
         # With an OffsetIndex the page's size (header included) is known, so read exactly that
         loc = @page_number && @locations[@page_number]
@@ -171,13 +217,18 @@ module Herringbone
         if buf.bytesize - off < size
           raise FormatError, "Column #{@column.dotted_path}: page overruns the file (read #{@seen} of #{@total} values)"
         end
-        body = off.zero? && buf.bytesize == size ? buf : buf.byteslice(off, size)
+        body = (off.zero? && buf.bytesize == size) ? buf : buf.byteslice(off, size)
         @pos += hlen + size
         [header, body]
       end
 
       # Returns [buffer, offset] where buffer[offset..] holds at least +need+ bytes from file
       # position +pos+ (fewer only at EOF), reading +len+ bytes when the buffer does not cover it.
+      #
+      # @param pos [Integer] file offset wanted
+      # @param len [Integer] bytes to read when the buffer has to be refilled
+      # @param need [Integer] bytes that must be available from +pos+ to reuse the buffer
+      # @return [Array(String, Integer)] binary buffer and the offset of +pos+ in it
       def window(pos, len, need = len)
         off = pos - @buf_pos
         return [@buf, off] if off >= 0 && off + need <= @buf.bytesize
@@ -188,12 +239,23 @@ module Herringbone
         [@buf, 0]
       end
 
+      # Decompresses a page body with the chunk's codec
+      #
+      # @param body [String] compressed bytes
+      # @param size [Integer] uncompressed size from the page header
+      # @return [String] uncompressed bytes
+      # @raise [UnsupportedError] for an unsupported codec, naming the column
       def decompress(body, size)
         Compression.decompress(@meta.codec, body, size)
       rescue UnsupportedError => e
         raise e, "#{e.message} (column #{@column.dotted_path})"
       end
 
+      # Decodes a dictionary page (always PLAIN) and keeps its converted values
+      #
+      # @param header [Format::PageHeader] the dictionary page header
+      # @param body [String] the compressed page body
+      # @return [Array] the dictionary values
       def read_dictionary(header, body)
         dh = header.dictionary_page_header
         data = decompress(body, header.uncompressed_page_size)
@@ -202,6 +264,11 @@ module Herringbone
         @dictionary = vals
       end
 
+      # A DATA_PAGE: the whole body is compressed, levels come first with their own length prefix
+      #
+      # @param header [Format::PageHeader] the data page header
+      # @param body [String] the compressed page body
+      # @return [PageStream::Page] the page, decoded on demand
       def data_page_v1(header, body)
         dh = header.data_page_header
         n = dh.num_values
@@ -214,6 +281,11 @@ module Herringbone
         PageStream::Page.new(n, defs, reps, values, conv)
       end
 
+      # A DATA_PAGE_V2: levels are stored uncompressed before the (optionally compressed) values
+      #
+      # @param header [Format::PageHeader] the data page header
+      # @param body [String] the page body
+      # @return [PageStream::Page] the page, decoded on demand
       def data_page_v2(header, body)
         dh = header.data_page_header_v2
         n = dh.num_values
@@ -230,9 +302,20 @@ module Herringbone
         PageStream::Page.new(n, defs, reps, values, conv)
       end
 
+      # Bit width of levels up to a max level (memoized Integer#bit_length)
       RLE_WIDTH = Hash.new { |h, k| h[k] = k.bit_length }
 
       # [decoder, position after the levels]
+      #
+      # @param data [String] the uncompressed page
+      # @param pos [Integer] offset of the levels in +data+
+      # @param encoding [Integer] Format::Encoding of the levels (RLE or BIT_PACKED)
+      # @param max [Integer] max level of the column, which sets the bit width
+      # @param n [Integer] number of entries in the page
+      # @return [Array(PageStream::HybridDecoder, Integer), Array(PageStream::ArrayDecoder, Integer)]
+      #   the decoder and the offset just past the levels
+      # @raise [FormatError] when the RLE length prefix is cut off
+      # @raise [UnsupportedError] for any other level encoding
       def level_decoder(data, pos, encoding, max, n)
         width = RLE_WIDTH[max]
         case encoding
@@ -248,11 +331,20 @@ module Herringbone
         end
       end
 
+      # Physical type => [String#unpack directive, byte width] for PLAIN fixed-width values
       FIXED_FORMATS = {
         T::INT32 => ["l<", 4], T::INT64 => ["q<", 8], T::FLOAT => ["e", 4], T::DOUBLE => ["E", 8]
       }.freeze
 
       # [value decoder, converter still to apply to its values (nil for dictionary pages)]
+      #
+      # @param data [String] the uncompressed values section
+      # @param pos [Integer] offset of the values in +data+
+      # @param encoding [Integer] Format::Encoding of the values
+      # @return [Array(Object, Proc)] a PageStream decoder (responds to #read) and the converter,
+      #   which may be nil
+      # @raise [FormatError] for a dictionary-encoded page in a chunk without a dictionary page
+      # @raise [UnsupportedError] for an encoding not valid for the column's type, or unknown
       def value_decoder(data, pos, encoding)
         type = @column.type
         decoder = case encoding
@@ -272,7 +364,7 @@ module Herringbone
           raise UnsupportedError, "RLE value encoding is only supported for BOOLEAN" unless type == T::BOOLEAN
           PageStream::RleBooleanDecoder.new(data, pos)
         when E::DELTA_BINARY_PACKED
-          bits = type == T::INT32 ? 32 : 64
+          bits = (type == T::INT32) ? 32 : 64
           PageStream::ArrayDecoder.new(Encodings::Delta.decode_binary_packed(data, pos, bits).first)
         when E::DELTA_LENGTH_BYTE_ARRAY
           PageStream::DeltaLengthDecoder.new(data, pos)

data/lib/herringbone/reader/column_cursor.rb

@@ -11,6 +11,7 @@ module Herringbone
       # Levels decoded ahead at a time when looking for row starts in repeated columns
       LOOKAHEAD = 4096
 
+      # @param chunk_reader [ColumnChunkReader] reader of the chunk, positioned at its start
       def initialize(chunk_reader)
         @src = chunk_reader
         col = chunk_reader.column
@@ -26,12 +27,17 @@ module Herringbone
         @page_idx = -1 # index of the current data page within the chunk
       end
 
-      # Rows handed out or skipped so far
+      # @return [Integer] rows handed out or skipped so far (the chunk row the cursor is at)
       attr_reader :row
 
       # Moves forward to row +target+ of the chunk (0-based). With an OffsetIndex, pages before
       # the one holding +target+ are not read at all; otherwise rows are skipped page by page,
       # decoding levels but not building values.
+      #
+      # @param target [Integer] row of the chunk to stop at
+      # @return [void]
+      # @raise [ArgumentError] when +target+ is before the current row
+      # @raise [FormatError] when the chunk runs out of pages before +target+
       def seek(target)
         raise ArgumentError, "Cannot seek backwards (at row #{@row}, asked for #{target})" if target < @row
         return if target == @row
@@ -53,6 +59,10 @@ module Herringbone
       end
 
       # Moves past the next +k+ rows without building their values
+      #
+      # @param k [Integer] number of rows to skip; zero or less does nothing
+      # @return [void]
+      # @raise [FormatError] when the chunk runs out of pages
       def skip(k)
         return if k <= 0
         @repeated ? take_repeated(k, false) : take_flat(k, false)
@@ -62,6 +72,11 @@ module Herringbone
 
       # [definition_levels, repetition_levels, values] of the next +k+ rows. Levels are nil
       # when the column's max level is 0.
+      #
+      # @param k [Integer] number of rows to hand out
+      # @return [Array(Array<Integer>, Array<Integer>, Array)] definition levels, repetition
+      #   levels and the (converted) values of the non-null entries
+      # @raise [FormatError] when the chunk runs out of pages or does not start at a row boundary
       def take(k)
         pieces = @repeated ? take_repeated(k) : take_flat(k)
         @row += k
@@ -79,6 +94,13 @@ module Herringbone
 
       private
 
+      # Non-repeated columns, where every entry is a row: reads page by page until +k+ entries
+      #
+      # @param k [Integer] number of rows
+      # @param keep [Boolean] false to skip the values instead of decoding them
+      # @return [Array<Array(Array<Integer>, nil, Array)>] [defs, nil, values] per page touched;
+      #   empty when +keep+ is false
+      # @raise [FormatError] when the chunk runs out of pages
       def take_flat(k, keep = true)
         pieces = []
         while k > 0
@@ -99,6 +121,12 @@ module Herringbone
 
       # Collects entries until +k+ rows have started and the next row start (or the end of the
       # column) is reached. Values are read from a page before moving on to the next one.
+      #
+      # @param k [Integer] number of rows
+      # @param keep [Boolean] false to skip the values instead of decoding them
+      # @return [Array<Array(Array<Integer>, Array<Integer>, Array)>] [defs, reps, values] per
+      #   page touched (defs nil without definition levels); empty when +keep+ is false
+      # @raise [FormatError] when the first page does not start at a row boundary
       def take_repeated(k, keep = true)
         pieces = []
         rows = 0
@@ -106,7 +134,7 @@ module Herringbone
         reps = []
         while true
           if @bi >= @br.to_a.size
-            if @page && @page.remaining.positive?
+            if @page&.remaining&.positive?
               @bd, @br = @page.read_levels(LOOKAHEAD)
               @bi = 0
             else
@@ -147,6 +175,12 @@ module Herringbone
       end
 
       # Reads (or skips) the values belonging to the collected entries of the current page
+      #
+      # @param pieces [Array<Array>] output list a [defs, reps, values] piece is appended to
+      # @param defs [Array<Integer>, nil] collected definition levels (nil without them)
+      # @param reps [Array<Integer>] collected repetition levels; nothing happens when empty
+      # @param keep [Boolean] false to skip the values instead of appending a piece
+      # @return [void]
       def flush(pieces, defs, reps, keep)
         return if reps.empty?
         nv = defs ? defs.count(@max_def) : reps.size
@@ -157,18 +191,28 @@ module Herringbone
         end
       end
 
+      # The next +n+ values of the current page, converted when the page has a converter
+      #
+      # @param n [Integer] number of values (non-null entries)
+      # @return [Array] Ruby values
       def values(n)
         vals = @page.read_values(n)
         conv = @page.converter
         conv ? vals.map!(&conv) : vals
       end
 
+      # Like #load_page, but running out of pages is an error
+      #
+      # @return [void]
+      # @raise [FormatError] when the chunk has no more data pages
       def load_page!
         return if load_page
         raise FormatError, "Column #{@path}: ran out of pages after #{@src.seen} of #{@src.total} values"
       end
 
       # Moves to the next data page; false at the end of the chunk
+      #
+      # @return [Boolean] whether a page was loaded
       def load_page
         @page = @src.next_stream or return false
         @page_idx += 1