herringbone 0.2.0 → 0.3.0

data/lib/herringbone/types.rb

@@ -21,19 +21,44 @@ module Herringbone
   module Types
     module_function
 
+    # Shorthand for Format::Type
     T = Format::Type
+    # Shorthand for Format::ConvertedType
     C = Format::ConvertedType
 
+    # Julian day number of 1970-01-01, the origin of DATE columns
     EPOCH_JD = Date.new(1970, 1, 1).jd
+    # Julian day number of 1970-01-01, the origin of the day half of an INT96 timestamp
     JULIAN_EPOCH_DAY = 2_440_588 # Julian day number of 1970-01-01, used by INT96
+    # Nanoseconds in a day, used to split INT96 timestamps into day and nanoseconds of day
     NANOS_PER_DAY = 86_400 * 1_000_000_000
 
-    UNIT_DIVISORS = { millis: 1_000, micros: 1_000_000, nanos: 1_000_000_000 }.freeze
-    UNIT_NAMES = { millis: :millisecond, micros: :microsecond, nanos: :nanosecond }.freeze
-
-
+    # Ticks per second for each TimeUnit
+    UNIT_DIVISORS = {millis: 1_000, micros: 1_000_000, nanos: 1_000_000_000}.freeze
+    # Sub-second unit names understood by +Time.at+, for each TimeUnit
+    UNIT_NAMES = {millis: :millisecond, micros: :microsecond, nanos: :nanosecond}.freeze
+
+    # Shorthand for building a LogicalType union.
+    # @param kw [Hash{Symbol => Thrift::Struct}] the single union member to set; any LogicalType
+    #   field is accepted, those used in this module are listed below
+    # @option kw [Format::StringType] :string
+    # @option kw [Format::EnumType] :enum
+    # @option kw [Format::JsonType] :json
+    # @option kw [Format::BsonType] :bson
+    # @option kw [Format::UUIDType] :uuid
+    # @option kw [Format::DateType] :date
+    # @option kw [Format::Float16Type] :float16
+    # @option kw [Format::IntType] :integer bit width and signedness
+    # @option kw [Format::DecimalType] :decimal scale and precision
+    # @option kw [Format::TimeType] :time unit and UTC adjustment
+    # @option kw [Format::TimestampType] :timestamp unit and UTC adjustment
+    # @return [Format::LogicalType]
     def lt(**kw) = Format::LogicalType.new(**kw)
 
+    # TimeUnit union for a unit name.
+    # @param unit [Symbol, String] +:millis+/+:ms+, +:micros+/+:us+ or +:nanos+/+:ns+
+    # @return [Format::TimeUnit]
+    # @raise [ArgumentError] for any other unit
     def time_unit(unit)
       case unit.to_sym
       when :millis, :ms then Format::TimeUnit.millis
@@ -43,64 +68,84 @@ module Herringbone
       end
     end
 
+    # Physical attributes of an annotated integer column, with both the INTEGER logical type
+    # and the matching INT_n/UINT_n converted type.
+    # @param bits [Integer] bit width: 8, 16, 32 or 64 (64 is stored as INT64, the rest as INT32)
+    # @param signed [Boolean] whether the integers are signed
+    # @return [Hash{Symbol => Object}] +:type+, +:converted_type+ and +:logical_type+
     def int_type(bits, signed)
-      physical = bits == 64 ? T::INT64 : T::INT32
+      physical = (bits == 64) ? T::INT64 : T::INT32
       converted = signed ? C.const_get("INT_#{bits}") : C.const_get("UINT_#{bits}")
-      { type: physical, converted_type: converted,
-        logical_type: lt(integer: Format::IntType.new(bit_width: bits, is_signed: signed)) }
+      {type: physical, converted_type: converted,
+       logical_type: lt(integer: Format::IntType.new(bit_width: bits, is_signed: signed))}
     end
 
     # Physical attributes (type, logical type etc.) for a DSL type name
+    # @param type [Symbol, String] column type name, e.g. +:string+, +:int64+, +:decimal+, +:timestamp+
+    # @param opts [Hash{Symbol => Object}] type-specific options
+    # @option opts [Boolean] :parquet_enum for +:enum+, annotate as ENUM instead of STRING
+    # @option opts [Integer] :length byte length for +:fixed+ (required)
+    # @option opts [Symbol] :unit (:micros) for +:time+ and +:timestamp+, see {time_unit}
+    # @option opts [Boolean] :utc (true) for +:time+ and +:timestamp+, sets +is_adjusted_to_utc+;
+    #   the legacy converted type is only written when true
+    # @option opts [Integer] :precision number of decimal digits for +:decimal+ (required)
+    # @option opts [Integer] :scale (0) digits after the decimal point for +:decimal+
+    # @option opts [Symbol] :physical for +:decimal+, force +:int32+, +:int64+, +:binary+ or
+    #   +:fixed+ storage instead of picking the smallest by precision
+    # @return [Hash{Symbol => Object}] keyword arguments for Schema::Node.new: +:type+ and,
+    #   where applicable, +:type_length+, +:converted_type+, +:logical_type+, +:scale+, +:precision+
+    # @raise [ArgumentError] for an unknown type, unit, or invalid decimal precision/scale
+    # @raise [KeyError] when a required option is missing or +:physical+ is not recognized
     def physical_attributes(type, **opts)
       case type.to_sym
-      when :boolean then { type: T::BOOLEAN }
+      when :boolean then {type: T::BOOLEAN}
       when :int8 then int_type(8, true)
       when :int16 then int_type(16, true)
-      when :int32 then { type: T::INT32 }
-      when :int64 then { type: T::INT64 }
+      when :int32 then {type: T::INT32}
+      when :int64 then {type: T::INT64}
       when :uint8 then int_type(8, false)
       when :uint16 then int_type(16, false)
       when :uint32 then int_type(32, false)
       when :uint64 then int_type(64, false)
-      when :float then { type: T::FLOAT }
-      when :double then { type: T::DOUBLE }
-      when :float16 then { type: T::FIXED_LEN_BYTE_ARRAY, type_length: 2, logical_type: lt(float16: Format::Float16Type.new) }
-      when :string then { type: T::BYTE_ARRAY, converted_type: C::UTF8, logical_type: lt(string: Format::StringType.new) }
-      when :binary then { type: T::BYTE_ARRAY }
-      when :json then { type: T::BYTE_ARRAY, converted_type: C::JSON, logical_type: lt(json: Format::JsonType.new) }
-      when :bson then { type: T::BYTE_ARRAY, converted_type: C::BSON, logical_type: lt(bson: Format::BsonType.new) }
+      when :float then {type: T::FLOAT}
+      when :double then {type: T::DOUBLE}
+      when :float16 then {type: T::FIXED_LEN_BYTE_ARRAY, type_length: 2, logical_type: lt(float16: Format::Float16Type.new)}
+      when :string then {type: T::BYTE_ARRAY, converted_type: C::UTF8, logical_type: lt(string: Format::StringType.new)}
+      when :binary then {type: T::BYTE_ARRAY}
+      when :json then {type: T::BYTE_ARRAY, converted_type: C::JSON, logical_type: lt(json: Format::JsonType.new)}
+      when :bson then {type: T::BYTE_ARRAY, converted_type: C::BSON, logical_type: lt(bson: Format::BsonType.new)}
       when :enum
         if opts[:parquet_enum]
-          { type: T::BYTE_ARRAY, converted_type: C::ENUM, logical_type: lt(enum: Format::EnumType.new) }
+          {type: T::BYTE_ARRAY, converted_type: C::ENUM, logical_type: lt(enum: Format::EnumType.new)}
         else
           physical_attributes(:string)
         end
-      when :uuid then { type: T::FIXED_LEN_BYTE_ARRAY, type_length: 16, logical_type: lt(uuid: Format::UUIDType.new) }
-      when :date then { type: T::INT32, converted_type: C::DATE, logical_type: lt(date: Format::DateType.new) }
-      when :int96 then { type: T::INT96 }
+      when :uuid then {type: T::FIXED_LEN_BYTE_ARRAY, type_length: 16, logical_type: lt(uuid: Format::UUIDType.new)}
+      when :date then {type: T::INT32, converted_type: C::DATE, logical_type: lt(date: Format::DateType.new)}
+      when :int96 then {type: T::INT96}
       when :fixed
-        { type: T::FIXED_LEN_BYTE_ARRAY, type_length: Integer(opts.fetch(:length)) }
+        {type: T::FIXED_LEN_BYTE_ARRAY, type_length: Integer(opts.fetch(:length))}
       when :time
         unit = time_unit(opts.fetch(:unit, :micros))
-        converted = { millis: C::TIME_MILLIS, micros: C::TIME_MICROS }[unit.to_sym]
-        { type: unit.millis ? T::INT32 : T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
-          logical_type: lt(time: Format::TimeType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit)) }
+        converted = {millis: C::TIME_MILLIS, micros: C::TIME_MICROS}[unit.to_sym]
+        {type: unit.millis ? T::INT32 : T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
+         logical_type: lt(time: Format::TimeType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit))}
       when :timestamp
         unit = time_unit(opts.fetch(:unit, :micros))
-        converted = { millis: C::TIMESTAMP_MILLIS, micros: C::TIMESTAMP_MICROS }[unit.to_sym]
-        { type: T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
-          logical_type: lt(timestamp: Format::TimestampType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit)) }
+        converted = {millis: C::TIMESTAMP_MILLIS, micros: C::TIMESTAMP_MICROS}[unit.to_sym]
+        {type: T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
+         logical_type: lt(timestamp: Format::TimestampType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit))}
       when :decimal
         precision = Integer(opts.fetch(:precision))
         scale = Integer(opts.fetch(:scale, 0))
         raise ArgumentError, "Decimal precision must be positive" unless precision.positive?
         raise ArgumentError, "Decimal scale must be between 0 and precision" unless scale.between?(0, precision)
         physical = if opts[:physical]
-          { int32: { type: T::INT32 }, int64: { type: T::INT64 }, binary: { type: T::BYTE_ARRAY },
-            fixed: { type: T::FIXED_LEN_BYTE_ARRAY, type_length: decimal_bytes(precision) } }.fetch(opts[:physical])
-        elsif precision <= 9 then { type: T::INT32 }
-        elsif precision <= 18 then { type: T::INT64 }
-        else { type: T::FIXED_LEN_BYTE_ARRAY, type_length: decimal_bytes(precision) }
+          {int32: {type: T::INT32}, int64: {type: T::INT64}, binary: {type: T::BYTE_ARRAY},
+           fixed: {type: T::FIXED_LEN_BYTE_ARRAY, type_length: decimal_bytes(precision)}}.fetch(opts[:physical])
+        elsif precision <= 9 then {type: T::INT32}
+        elsif precision <= 18 then {type: T::INT64}
+        else {type: T::FIXED_LEN_BYTE_ARRAY, type_length: decimal_bytes(precision)}
         end
         physical.merge(converted_type: C::DECIMAL, scale: scale, precision: precision,
           logical_type: lt(decimal: Format::DecimalType.new(scale: scale, precision: precision)))
@@ -110,6 +155,8 @@ module Herringbone
     end
 
     # Minimal number of bytes to hold a signed integer of +precision+ decimal digits
+    # @param precision [Integer] number of decimal digits
+    # @return [Integer] byte length for a FIXED_LEN_BYTE_ARRAY decimal
     def decimal_bytes(precision)
       max = 10**precision
       n = 1
@@ -118,6 +165,17 @@ module Herringbone
     end
 
     # Normalized logical kind of a node: [symbol, details]
+    #
+    # The LogicalType wins when present; otherwise the legacy converted type is mapped onto
+    # the same shapes. Details depend on the kind:
+    #   [:integer, bit_width, signed]
+    #   [:decimal, scale, precision]
+    #   [:timestamp, unit, adjusted_to_utc]   (unit is :millis, :micros or :nanos)
+    #   [:time, unit, adjusted_to_utc]
+    #   [kind]                                for other annotations (:string, :date, :uuid ...)
+    #   [nil]                                 for unannotated columns
+    # @param node [Schema::Node] leaf node of the physical schema
+    # @return [Array] kind Symbol (or nil) followed by its details
     def logical_of(node)
       if (kind = node.logical_type&.kind)
         name, payload = kind
@@ -153,6 +211,8 @@ module Herringbone
     end
 
     # Returns a lambda converting a physical value into a Ruby value, or nil when no conversion is needed.
+    # @param node [Schema::Node] leaf node of the physical schema
+    # @return [Proc, nil] one-argument converter, or nil when decoded values are used as-is
     def reader_for(node)
       kind, a, b = logical_of(node)
       type = node.type
@@ -161,7 +221,7 @@ module Herringbone
         return ->(v) { v.force_encoding(Encoding::UTF_8) } if type == T::BYTE_ARRAY || type == T::FIXED_LEN_BYTE_ARRAY
       when :integer
         if !b && (type == T::INT32 || type == T::INT64)
-          mask = type == T::INT32 ? 0xFFFF_FFFF : 0xFFFF_FFFF_FFFF_FFFF
+          mask = (type == T::INT32) ? 0xFFFF_FFFF : 0xFFFF_FFFF_FFFF_FFFF
           return ->(v) { v & mask }
         end
       when :date
@@ -179,12 +239,18 @@ module Herringbone
       nil
     end
 
+    # Converter from an INT64 timestamp to a UTC Time.
+    # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
+    # @return [Proc] lambda taking an Integer count of +unit+ since the epoch
+    # @raise [KeyError] for an unknown unit
     def timestamp_reader(unit)
       div = UNIT_DIVISORS.fetch(unit)
       name = UNIT_NAMES.fetch(unit)
       ->(v) { Time.at(v / div, v % div, name).utc }
     end
 
+    # Converter from a decoded INT96 value to a UTC Time.
+    # @return [Proc] lambda taking a +[nanoseconds_of_day, julian_day]+ pair
     def int96_reader
       lambda do |(nanos, day)|
         secs = (day - JULIAN_EPOCH_DAY) * 86_400
@@ -192,6 +258,11 @@ module Herringbone
       end
     end
 
+    # Converter from a stored unscaled decimal to a BigDecimal.
+    # @param type [Integer] physical type (Format::Type) of the column
+    # @param scale [Integer] digits after the decimal point
+    # @return [Proc, nil] lambda taking an Integer (INT32/INT64) or big-endian two's complement
+    #   bytes (BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY), or nil for any other physical type
     def decimal_reader(type, scale)
       to_decimal = scale.zero? ? ->(i) { BigDecimal(i) } : ->(i) { BigDecimal("#{i}e-#{scale}") }
       case type
@@ -202,13 +273,20 @@ module Herringbone
     end
 
     # Big-endian two's complement bytes to Integer
+    # @param bytes [String] binary string; empty decodes as 0
+    # @return [Integer]
     def be_to_int(bytes)
       return 0 if bytes.empty?
       i = bytes.unpack1("H*").to_i(16)
       bits = bytes.bytesize * 8
-      i >= (1 << (bits - 1)) ? i - (1 << bits) : i
+      (i >= (1 << (bits - 1))) ? i - (1 << bits) : i
     end
 
+    # Integer to big-endian two's complement bytes of a fixed width
+    # @param i [Integer] value to encode
+    # @param nbytes [Integer] output width in bytes
+    # @return [String] binary string of +nbytes+ bytes
+    # @raise [EncodeError] when +i+ does not fit in +nbytes+ bytes
     def int_to_be(i, nbytes)
       bits = nbytes * 8
       raise EncodeError, "Decimal value #{i} does not fit in #{nbytes} bytes" unless i.bit_length < bits
@@ -216,6 +294,9 @@ module Herringbone
       [i.to_s(16).rjust(nbytes * 2, "0")].pack("H*")
     end
 
+    # Decodes an IEEE 754 half-precision value, including subnormals, infinities and NaN.
+    # @param h [Integer] 16-bit pattern
+    # @return [Float]
     def half_to_float(h)
       sign = (h >> 15).zero? ? 1.0 : -1.0
       exp = (h >> 10) & 0x1F
@@ -231,9 +312,11 @@ module Herringbone
 
     # Rounds a Float to the nearest half-precision value (ties to even). Every step is exact
     # in double arithmetic, so there is no double rounding.
+    # @param f [Float] value to round; overflows to infinity, NaN becomes the canonical quiet NaN
+    # @return [Integer] 16-bit pattern
     def float_to_half(f)
       return 0x7E00 if f.nan?
-      sign = f.negative? || (f.zero? && (1.0 / f).negative?) ? 0x8000 : 0
+      sign = (f.negative? || (f.zero? && (1.0 / f).negative?)) ? 0x8000 : 0
       a = f.abs
       return sign | 0x7C00 if a >= 65_520.0
       return sign | (a * 2.0**24).round(half: :even) if a < 2.0**-14
@@ -252,13 +335,17 @@ module Herringbone
     #   date:      Date, Time/DateTime (its calendar date), ISO-8601 String, Integer days since epoch
     #   timestamp: Time, DateTime, ActiveSupport::TimeWithZone, Date (midnight UTC), ISO-8601 String,
     #              Integer in the column's unit
-    #   time:      Time (its time of day), "HH:MM:SS[.fraction]" String, Integer in the column's unit
+    #   time:      Time (its time of day), "HH:MM[:SS[.fraction]]" String, Integer in the column's unit
     #   json:      String (used as-is) or any other object (serialized with JSON.generate)
     #   string:    String, Symbol or anything responding to to_s
     #   boolean:   true/false, 1/0, "true"/"false", "t"/"f", "1"/"0", "yes"/"no"
     #   integers:  Integer, or a Float/BigDecimal/Rational/String holding a whole number
     #   decimal:   BigDecimal, Integer, Rational, Float or numeric String
     #   uuid:      String with or without dashes, or 16 raw bytes
+    # The returned lambda raises ArgumentError or RangeError for values it cannot convert.
+    # @param node [Schema::Node] leaf node of the physical schema
+    # @return [Proc, nil] one-argument converter; nil only for a column with no recognized
+    #   physical type
     def writer_for(node)
       kind, a, b = logical_of(node)
       type = node.type
@@ -279,13 +366,13 @@ module Herringbone
         return decimal_writer(node, a)
       when :uuid
         return fixed_checker(16) do |v|
-          v.bytesize == 16 && v.encoding == Encoding::BINARY ? v : [v.to_s.delete("-")].pack("H*")
+          (v.bytesize == 16 && v.encoding == Encoding::BINARY) ? v : [v.to_s.delete("-")].pack("H*")
         end
       when :float16
         return ->(v) { [float_to_half(Float(v))].pack("v") }
       when :integer
         if !b && (type == T::INT32 || type == T::INT64)
-          bits = type == T::INT32 ? 32 : 64
+          bits = (type == T::INT32) ? 32 : 64
           check = int_checker(0, (1 << a) - 1)
           return ->(v) { Encodings::Delta.wrap(check.call(v), bits) }
         elsif type == T::INT32 || type == T::INT64
@@ -311,19 +398,29 @@ module Herringbone
       end
     end
 
+    # Values accepted for a boolean column (strings are also matched case-insensitively)
     BOOLEANS = {
       true => true, false => false, 1 => true, 0 => false,
       "true" => true, "false" => false, "t" => true, "f" => false, "1" => true, "0" => false,
       "yes" => true, "no" => false, "TRUE" => true, "FALSE" => false, "T" => true, "F" => false
     }.freeze
 
+    # Coerces a value to true/false using BOOLEANS.
+    # @param v [Object] true/false, 1/0, or a String/Symbol such as "yes" or :false
+    # @return [Boolean]
+    # @raise [ArgumentError] when +v+ is not recognized as a boolean
     def to_boolean(v)
       BOOLEANS.fetch(v) do
-        s = v.is_a?(String) || v.is_a?(Symbol) ? v.to_s.downcase : nil
+        s = (v.is_a?(String) || v.is_a?(Symbol)) ? v.to_s.downcase : nil
         BOOLEANS.fetch(s) { raise ArgumentError, "expected a boolean, got #{v.inspect}" }
       end
     end
 
+    # Days since the Unix epoch in the proleptic Gregorian calendar, as stored in DATE columns.
+    # @param v [Date, String, Integer, #to_date] a Date, an ISO-8601 date String, an Integer
+    #   (returned as-is) or anything responding to +to_date+ (Time, DateTime)
+    # @return [Integer]
+    # @raise [ArgumentError] when +v+ cannot be turned into a Date
     def date_to_days(v)
       return v if v.is_a?(Integer)
       d = case v
@@ -338,6 +435,10 @@ module Herringbone
     end
 
     # Converts the values a timestamp column accepts into a Time (or Time-like) object
+    # @param v [Time, DateTime, Date, String, #to_i] a Time, DateTime, Date (taken as midnight UTC),
+    #   ISO-8601 (or +Time.parse+-able) String, or a Time-like object responding to +to_i+ and +nsec+
+    # @return [Time, Object] a Time, or +v+ itself when it is Time-like
+    # @raise [ArgumentError] when +v+ is none of the above or the String cannot be parsed
     def to_time(v)
       case v
       when Time then v
@@ -356,6 +457,13 @@ module Herringbone
       end
     end
 
+    # Converter from a timestamp-like value to an INT64 count of +unit+ since the epoch.
+    # Integers pass through unchanged; sub-unit precision is truncated.
+    # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
+    # @param utc [Boolean] whether the column is adjusted to UTC; when false the local wall
+    #   clock time of the value (its UTC offset added) is stored
+    # @return [Proc] one-argument lambda, see {to_time} for accepted values
+    # @raise [KeyError] for an unknown unit
     def timestamp_writer(unit, utc)
       mult = UNIT_DIVISORS.fetch(unit)
       lambda do |v|
@@ -368,6 +476,13 @@ module Herringbone
       end
     end
 
+    # Converter from a time of day to a count of +unit+ since midnight.
+    # The lambda accepts an Integer (returned as-is), an "HH:MM[:SS[.fraction]]" String or
+    # anything responding to +hour+ and +nsec+ (Time, DateTime), and raises ArgumentError or
+    # RangeError for anything else or a time past 23:59:59.
+    # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
+    # @return [Proc] one-argument lambda
+    # @raise [KeyError] for an unknown unit
     def time_writer(unit)
       mult = UNIT_DIVISORS.fetch(unit)
       lambda do |v|
@@ -389,6 +504,9 @@ module Herringbone
 
     # String column restricted to a set of values. +values+ is an Array of labels, or a Hash of
     # label => stored value like Rails' `Model.statuses`, in which case either is accepted.
+    # Symbols are accepted in place of String labels.
+    # @param values [Array<String, Symbol>, Hash{String, Symbol => Object}] allowed labels
+    # @return [Proc] lambda returning the String label, raising ArgumentError for any other value
     def enum_writer(values)
       labels = {}
       if values.is_a?(Hash)
@@ -407,15 +525,24 @@ module Herringbone
     end
 
     # Converts to Integer, rejecting fractional numbers and values outside min..max
+    # @param min [Integer] smallest accepted value
+    # @param max [Integer] largest accepted value
+    # @return [Proc] lambda raising ArgumentError for non-integers and RangeError when out of range
     def int_checker(min, max)
       lambda do |v|
         i = Integer(v)
         raise ArgumentError, "#{v.inspect} is not an integer" unless v.is_a?(Integer) || !v.is_a?(Numeric) || v == i
-        raise RangeError, "#{i} is outside #{min}..#{max}" unless i >= min && i <= max
+        raise RangeError, "#{i} is outside #{min}..#{max}" unless i.between?(min, max)
         i
       end
     end
 
+    # Wraps a conversion to a String and checks the result has exactly +length+ bytes.
+    # @param length [Integer] required byte length
+    # @yield [v] converts the incoming value
+    # @yieldparam v [Object] value handed to the writer
+    # @yieldreturn [String] bytes to store
+    # @return [Proc] lambda raising ArgumentError when the converted String has another length
     def fixed_checker(length, &convert)
       lambda do |v|
         s = convert.call(v)
@@ -424,6 +551,14 @@ module Herringbone
       end
     end
 
+    # Converter from a numeric value to the stored unscaled decimal. Values are rounded
+    # (half away from zero) to +scale+ digits; the lambda raises RangeError when the result
+    # exceeds the column's precision.
+    # @param node [Schema::Node] DECIMAL leaf node, for its physical type, precision and type length
+    # @param scale [Integer] digits after the decimal point
+    # @return [Proc, nil] lambda returning an Integer (INT32/INT64) or big-endian two's complement
+    #   bytes (FIXED_LEN_BYTE_ARRAY, BYTE_ARRAY with minimal length), or nil for any other
+    #   physical type
     def decimal_writer(node, scale)
       mult = 10**scale
       limit = node.precision ? 10**node.precision : nil

data/lib/herringbone/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module Herringbone
-  VERSION = "0.2.0"
+  # Gem version, also written to the +created_by+ field of every file's footer
+  VERSION = "0.3.0"
 end

data/lib/herringbone/visualizer.rb

@@ -16,6 +16,7 @@ module Herringbone
   #
   #   File.open("data.parquet", "rb") { |io| Herringbone::Inspector.new(io).to_html }
   class Visualizer
+    # highlight.js build loaded by the page to colour the footer JSON; optional.
     HIGHLIGHT_JS = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"
     # The design and idea of this page come from Parquet X-ray; credited at the top of every page
     CREDIT_URL = "https://huggingface.co/spaces/cfahlgren1/parquet-xray"
@@ -26,9 +27,15 @@ module Herringbone
     # Footer JSON is embedded only when the footer is smaller than this
     MAX_FOOTER_JSON = 512 * 1024
 
-    PAGE_TYPES = { DATA_PAGE: 0, INDEX_PAGE: 1, DICTIONARY_PAGE: 2, DATA_PAGE_V2: 3 }.freeze
+    # PageType names to their Thrift ids, the page type code the embedded JS expects in page rows
+    # (unknown page types are passed on as their raw Integer).
+    PAGE_TYPES = {DATA_PAGE: 0, INDEX_PAGE: 1, DICTIONARY_PAGE: 2, DATA_PAGE_V2: 3}.freeze
 
     # +inspector+ is an Inspector. +title+ defaults to the file's name.
+    # @param inspector [Inspector] inspector over the file to render
+    # @param title [String, nil] page title; defaults to the file's name
+    # @param max_pages [Integer] page budget for per-page detail (see MAX_PAGES)
+    # @raise [ArgumentError] if +inspector+ is not an Inspector
     def initialize(inspector, title: nil, max_pages: MAX_PAGES)
       raise ArgumentError, "Expected a Herringbone::Inspector, got #{inspector.class}" unless inspector.is_a?(Inspector)
       @inspector = inspector
@@ -36,11 +43,13 @@ module Herringbone
       @max_pages = max_pages
     end
 
+    # Builds the page. Walks every page header of the file (and reads the page indexes).
+    # @return [String] a complete, self-contained HTML document
     def to_html
       json = JSON.generate(payload).gsub("<", "\\u003c").gsub("\u2028", "\\u2028").gsub("\u2029", "\\u2029")
       name = @title || @inspector.name || "Parquet file"
-      values = { "TITLE" => escape_html("#{name} · Parquet layout"), "HIGHLIGHT_JS" => HIGHLIGHT_JS,
-                 "CREDIT_URL" => CREDIT_URL, "DATA" => json }
+      values = {"TITLE" => escape_html("#{name} · Parquet layout"), "HIGHLIGHT_JS" => HIGHLIGHT_JS,
+                "CREDIT_URL" => CREDIT_URL, "DATA" => json}
       # One pass, so placeholder-like text in the data is never substituted
       TEMPLATE.gsub(/%%(TITLE|HIGHLIGHT_JS|CREDIT_URL|DATA)%%/) { values.fetch(Regexp.last_match(1)) }
     end
@@ -48,6 +57,8 @@ module Herringbone
     private
 
     # The data embedded in the page
+    # @return [Hash{Symbol => Object}] file, schema, columns, row groups (with chunks and pages),
+    #   key/value metadata and footer JSON, ready for JSON.generate
     def payload
       i = @inspector
       budget = @max_pages
@@ -75,6 +86,8 @@ module Herringbone
       }
     end
 
+    # Inspector#summary plus the display name and whether per-page detail was truncated.
+    # @return [Hash{Symbol => Object}] file-level facts for the page header
     def file_info
       s = @inspector.summary
       name = @title || @inspector.name || "(IO)"
@@ -82,6 +95,10 @@ module Herringbone
         index_mismatches: @inspector.column_chunks.sum { |c| c.index_mismatches.size })
     end
 
+    # One leaf column with its totals over all row groups.
+    # @param col [Schema::Column] leaf column
+    # @param t [Hash{Symbol => Object}] the column's entry from Inspector#column_totals
+    # @return [Hash{Symbol => Object}] column row, keyed with the short names the JS uses
     def column_info(col, t)
       node = col.node
       {
@@ -96,6 +113,11 @@ module Herringbone
       }
     end
 
+    # One column chunk: its metadata, statistics and index ranges and, when +with_pages+, its page
+    # rows, ColumnIndex and OffsetIndex.
+    # @param c [Inspector::ColumnChunkInfo] chunk to describe
+    # @param with_pages [Boolean] whether to include per-page detail (false once over the page budget)
+    # @return [Hash{Symbol => Object}] JSON-safe chunk entry, nil members omitted
     def chunk_info(c, with_pages)
       st = c.statistics
       h = {
@@ -106,8 +128,8 @@ module Herringbone
         stats: st && stats_info(st),
         bloom: c.bloom_filter_offset && [c.bloom_filter_offset, c.bloom_filter_length],
         ci: c.column_index_range, oi: c.offset_index_range,
-        dict: c.dictionary_page && { n: c.dictionary_page.num_values, cs: c.dictionary_page.compressed_size,
-                                     us: c.dictionary_page.uncompressed_size, sorted: c.dictionary_page.is_sorted },
+        dict: c.dictionary_page && {n: c.dictionary_page.num_values, cs: c.dictionary_page.compressed_size,
+                                    us: c.dictionary_page.uncompressed_size, sorted: c.dictionary_page.is_sorted},
         np: c.pages.size, ndp: c.data_pages.size,
         size_stats: c.size_statistics,
         kv: c.key_value_metadata.empty? ? nil : c.key_value_metadata,
@@ -133,6 +155,8 @@ module Herringbone
       Inspector.jsonable(h.compact)
     end
 
+    # @param st [Inspector::Stats] chunk statistics
+    # @return [Hash{Symbol => Object}] statistics with min/max in display form, nil members omitted
     def stats_info(st)
       {
         min: disp(st.min), max: disp(st.max), nulls: st.null_count, distinct: st.distinct_count,
@@ -142,24 +166,30 @@ module Herringbone
 
     # [type, offset, header_size, compressed, uncompressed, values, nulls, rows, first_row, encoding,
     #  min, max, crc, extra]; crc is 0 (none), 1 (present, not verified), 2 (verified ok) or 3 (mismatch)
+    # @param p [Inspector::PageInfo] page header to describe
+    # @return [Array] the page row, compact so pages of large files stay small
     def page_row(p)
       st = p.statistics
       extra = if p.type == :DATA_PAGE_V2
         "levels #{p.repetition_levels_byte_length}+#{p.definition_levels_byte_length} B" \
-          "#{p.is_compressed ? "" : ", not compressed"}"
+          "#{", not compressed" unless p.is_compressed}"
       elsif p.type == :DATA_PAGE
         [p.definition_level_encoding && "def #{p.definition_level_encoding}",
           p.repetition_level_encoding && "rep #{p.repetition_level_encoding}"].compact.join(", ")
       elsif p.dictionary?
         p.is_sorted ? "sorted" : nil
       end
-      [PAGE_TYPES.fetch(p.type, 1), p.offset, p.header_size, p.compressed_size, p.uncompressed_size, p.num_values,
+      [PAGE_TYPES.fetch(p.type) { p.type }, p.offset, p.header_size, p.compressed_size, p.uncompressed_size, p.num_values,
         p.num_nulls, p.num_rows, p.first_row_index, p.encoding, st && disp(st.min), st && disp(st.max),
         CRC_STATES.fetch(p.checksum) { p.crc.nil? ? 0 : 1 }, extra]
     end
 
-    CRC_STATES = { absent: 0, ok: 2, mismatch: 3 }.freeze
+    # PageInfo#checksum states to the crc codes of page_row (1, present but unverified, is the fallback).
+    CRC_STATES = {absent: 0, ok: 2, mismatch: 3}.freeze
 
+    # A page statistics vs ColumnIndex disagreement as a compact row.
+    # @param m [Hash{Symbol => Object}] entry from Inspector::ColumnChunkInfo#index_mismatches
+    # @return [Array] [page, field, value in page header, value in column index]; min/max in display form
     def mismatch_row(m)
       values = [m[:page_value], m[:index_value]]
       values = values.map { |v| disp(v) } if m[:field] == :min || m[:field] == :max
@@ -167,12 +197,14 @@ module Herringbone
     end
 
     # Display form of a decoded statistics value: strings quoted, the rest as text
+    # @param v [Object, nil] decoded value (String, numeric, Time, Date, Array, ...)
+    # @return [String, nil] at most 160 characters, nil for nil
     def disp(v)
       s = case v
       when nil then return nil
       when String
         t = Inspector.text(v)
-        v.encoding == Encoding::BINARY && t.start_with?("0x") ? t : JSON.generate(t)
+        (v.encoding == Encoding::BINARY && t.start_with?("0x")) ? t : JSON.generate(t)
       when Float then v.nan? ? "NaN" : v.to_s
       when BigDecimal then v.to_s("F")
       when Time then Inspector.jsonable(v)
@@ -180,30 +212,38 @@ module Herringbone
       when Array then JSON.generate(Inspector.jsonable(v))
       else v.to_s
       end
-      s.size > 160 ? "#{s[0, 159]}…" : s
+      (s.size > 160) ? "#{s[0, 159]}…" : s
     end
 
+    # The footer FileMetaData pretty-printed, or nil when it exceeds MAX_FOOTER_JSON bytes.
+    # @return [String, nil] JSON text
     def footer_json
       return nil if @inspector.footer_size > MAX_FOOTER_JSON
       JSON.pretty_generate(Inspector.jsonable(raw(@inspector.metadata.to_h)))
     end
 
     # Footer structs as plain data; binary statistics as hex, long strings shortened
+    # @param v [Object] a value from FileMetaData#to_h (Hash, Array, String or scalar)
+    # @return [Object] +v+ with every String passed through Inspector.text and cut at 300 characters
     def raw(v)
       case v
       when Hash then v.to_h { |k, x| [k, raw(x)] }
       when Array then v.map { |x| raw(x) }
       when String
         t = Inspector.text(v)
-        t.size > 300 ? "#{t[0, 300]}… (#{v.bytesize} bytes)" : t
+        (t.size > 300) ? "#{t[0, 300]}… (#{v.bytesize} bytes)" : t
       else v
       end
     end
 
+    # @param s [Object] text to put into HTML (converted with to_s)
+    # @return [String] +s+ with &, <, > and double quotes escaped
     def escape_html(s)
       s.to_s.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;").gsub('"', "&quot;")
     end
 
+    # The page: HTML, CSS and JS, with %%TITLE%%, %%HIGHLIGHT_JS%%, %%CREDIT_URL%% and %%DATA%%
+    # placeholders filled in by to_html.
     TEMPLATE = <<~'HTML'
       <!doctype html>
       <html lang="en">