herringbone 0.7.0 → 0.9.0

data/lib/herringbone/compression.rb

@@ -4,7 +4,8 @@ require "zlib"
 require "stringio"
 
 module Herringbone
-  # Raised when a file uses (or a writer asks for) a codec whose library is not loaded
+  # Raised when a file uses (or a writer asks for) a codec whose library is not loaded. The message
+  # says which gem to install, and that it has to be required before the read or write.
   class MissingCodecError < UnsupportedError
     # @return [String] codec name (+"ZSTD"+) and name of the gem providing it (+"zstd-ruby"+)
     attr_reader :codec, :gem_name
@@ -12,18 +13,37 @@ module Herringbone
     # @param codec [String] codec name, as shown in the message
     # @param gem_name [String] gem to add to the Gemfile
     # @param path [String] what to require to load the gem
-    def initialize(codec, gem_name, path)
+    # @param column [String, nil] dotted path of the column being read, when there is one
+    def initialize(codec, gem_name, path, column: nil)
       @codec = codec
       @gem_name = gem_name
-      super("#{codec} compression needs the \"#{gem_name}\" gem, which is not loaded. Add " \
-        "`gem \"#{gem_name}\"` to your Gemfile and `require \"#{path}\"` to use #{codec}.")
+      @path = path
+      where = column ? " to decompress column #{column}" : ""
+      super(<<~MESSAGE.chomp)
+        #{codec} compression needs the "#{gem_name}" gem#{where}, but it is not loaded.
+        Herringbone does not load optional codec gems by itself. To use #{codec}:
+
+          1. Install the gem: add `gem "#{gem_name}"` to your Gemfile, or run `gem install #{gem_name}`.
+          2. Load it with `require "#{path}"` before the read or write that needs #{codec},
+             for instance at the top of your script or in an initializer.
+      MESSAGE
+    end
+
+    # The same error, naming the column that could not be decompressed
+    #
+    # @param column [String] dotted path of the column
+    # @return [MissingCodecError]
+    # @api private
+    def for_column(column)
+      self.class.new(@codec, @gem_name, @path, column: column).tap { |e| e.set_backtrace(backtrace) if backtrace }
     end
   end
 
   # Dispatches page (de)compression by Parquet codec id. Snappy and LZ4 are pure Ruby and GZIP
   # uses zlib, so those always work. LZO is pure Ruby too, but can only be read. ZSTD and Brotli
   # come from optional gems (zstd-ruby, brotli), which the application requires; if they are
-  # not loaded, MissingCodecError says what to add.
+  # not loaded, MissingCodecError says what to add. ZSTD is the exception when reading: without
+  # zstd-ruby, pages are decompressed by Codecs::Zstd, in pure Ruby.
   # (Snappy also uses the optional snappy gem when it is loaded, see Codecs::Snappy.)
   #
   # @api private
@@ -137,7 +157,7 @@ module Herringbone
       when Format::Codec::LZ4_RAW then Codecs::LZ4.decompress_block(data, uncompressed_size)
       when Format::Codec::LZ4 then Codecs::LZ4.decompress_hadoop(data, uncompressed_size)
       when Format::Codec::LZO then Codecs::LZO.decompress_hadoop(data, uncompressed_size)
-      when Format::Codec::ZSTD then library(codec).decompress(data)
+      when Format::Codec::ZSTD then unzstd(data, uncompressed_size)
       when Format::Codec::BROTLI then library(codec).inflate(data)
       else
         raise UnsupportedError, "Unsupported compression codec #{Format::Codec::NAMES[codec] || codec}"
@@ -204,6 +224,15 @@ module Herringbone
       stream.finish
     end
 
+    # @param data [String] one or more Zstandard frames
+    # @param uncompressed_size [Integer] expected decompressed size in bytes
+    # @return [String] decompressed bytes, from zstd-ruby when it is loaded, else pure Ruby
+    # @raise [Codecs::Zstd::Error] on corrupt data, when decompressed in pure Ruby
+    def unzstd(data, uncompressed_size)
+      zstd = loaded_library(:Zstd)
+      zstd ? zstd.decompress(data) : Codecs::Zstd.decompress(data, uncompressed_size)
+    end
+
     # Codec output relabelled as binary without copying it, unless it is frozen
     #
     # @param str [String] bytes

data/lib/herringbone/inferring_writer.rb

@@ -158,7 +158,7 @@ module Herringbone
       type = case kind
       when nil then Format::Type::NAMES[column.type].downcase
       when :integer then "#{details[1] ? "int" : "uint"}#{details[0]}"
-      when :timestamp, :time then "#{kind} (#{details[0]})"
+      when :timestamp, :time then "#{"local_" unless details[1]}#{kind} (#{details[0]})"
       when :decimal then "decimal(#{details[1]}, #{details[0]})"
       else kind.to_s
       end

data/lib/herringbone/reader/column_chunk_reader.rb

@@ -303,6 +303,8 @@ module Herringbone
       # @raise [UnsupportedError] for an unsupported codec, naming the column
       def decompress(body, size)
         Compression.decompress(@meta.codec, body, size)
+      rescue MissingCodecError => e
+        raise e.for_column(@column.dotted_path)
       rescue UnsupportedError => e
         raise e, "#{e.message} (column #{@column.dotted_path})"
       end

data/lib/herringbone/schema/merge.rb

@@ -283,7 +283,7 @@ module Herringbone
         in [:int, 64, true] then Types.physical_attributes(:int64)
         in [:int, bits, signed] then Types.int_type(bits, signed)
         in [:float, bits] then Types.physical_attributes({16 => :float16, 32 => :float, 64 => :double}.fetch(bits))
-        in [:timestamp | :time => kind, unit, utc] then Types.physical_attributes(kind, unit: unit, utc: utc)
+        in [:timestamp | :time => kind, unit, utc] then Types.physical_attributes(utc ? kind : :"local_#{kind}", unit: unit)
         in [:text, :enum] then Types.physical_attributes(:enum, parquet_enum: true)
         in [:text, kind] then Types.physical_attributes(kind)
         in [:decimal, precision, scale] then Types.physical_attributes(:decimal, precision: precision, scale: scale)

data/lib/herringbone/schema.rb

@@ -470,6 +470,30 @@ module Herringbone
     end
     alias_method :&, :intersect
 
+    # The given top-level fields, in the order of this schema.
+    #
+    #   schema.only(:id, :name).fields.map(&:name) # => ["id", "name"]
+    #
+    # @param names [Array<String, Symbol>] names of top-level fields
+    # @return [Schema] a new schema, sharing no nodes with this one
+    # @raise [ArgumentError] when a name is not a top-level field, or no name is given
+    def only(*names)
+      keep = field_names!(names, "only")
+      subset { |f| keep.include?(f.name) }
+    end
+
+    # Every top-level field but the given ones, in the order of this schema.
+    #
+    #   schema.except(:ssn).fields.map(&:name) # => ["id", "name"]
+    #
+    # @param names [Array<String, Symbol>] names of top-level fields
+    # @return [Schema] a new schema, sharing no nodes with this one
+    # @raise [ArgumentError] when a name is not a top-level field, or all fields would be left out
+    def except(*names)
+      drop = field_names!(names, "except")
+      subset { |f| !drop.include?(f.name) }
+    end
+
     # @return [String] one line per node, indented by depth: repetition, physical type, name and
     #   annotation
     def inspect
@@ -501,6 +525,40 @@ module Herringbone
 
     private
 
+    # @param names [Array<String, Symbol, Array>] names given to #only or #except
+    # @param method [String] the method, for the error message
+    # @return [Array<String>] the names as Strings
+    # @raise [ArgumentError] when a name is not a top-level field, or no name is given
+    def field_names!(names, method)
+      names = names.flatten.map(&:to_s)
+      raise ArgumentError, "Schema##{method} needs the names of top-level fields, e.g. schema.#{method}(:id, :name)" if names.empty?
+      missing = names - @fields.map(&:name)
+      unless missing.empty?
+        raise ArgumentError, "Schema##{method}: no top-level field #{missing.join(", ")}. " \
+          "The schema has #{@fields.map(&:name).join(", ")}"
+      end
+      names
+    end
+
+    # @yieldparam field [Field] a top-level field
+    # @yieldreturn [Boolean] whether to keep it
+    # @return [Schema] a schema of copies of the kept fields' nodes
+    # @raise [ArgumentError] when no field is kept
+    def subset
+      nodes = @fields.select { |f| yield f }.map { |f| copy_node(f.node) }
+      raise ArgumentError, "A schema needs at least one field, and this would leave out all of #{@fields.map(&:name).join(", ")}" if nodes.empty?
+      Schema.new(Node.new(name: @root.name, repetition: :required, children: nodes))
+    end
+
+    # @param node [Node] node to copy, with its children
+    # @return [Node] a copy sharing no nodes with the original
+    def copy_node(node)
+      Node.new(name: node.name, repetition: node.repetition, type: node.type, type_length: node.type_length,
+        converted_type: node.converted_type, logical_type: node.logical_type, scale: node.scale,
+        precision: node.precision, field_id: node.field_id, enum_values: node.enum_values,
+        children: node.children&.map { |c| copy_node(c) })
+    end
+
     # Appends a Column for every leaf under +node+, depth-first. A non-required node adds one
     # definition level and a repeated node one repetition level.
     #
@@ -653,28 +711,50 @@ module Herringbone
         define_method(t) { |name, **opts| column(name, t, **opts) }
       end
 
-      # A TIME column; millis are stored as INT32, micros and nanos as INT64.
+      # A TIME column. A Time is converted to UTC when written, and its time of day in UTC is
+      # stored. Millis are stored as INT32, micros and nanos as INT64.
       #
       # @param name [String, Symbol] field name
       # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
-      # @param utc [Boolean] the isAdjustedToUTC flag of the logical type
       # @param opts [Hash{Symbol => Object}] options of #column
       # @option opts [Boolean] :null (true) whether the field is nullable
       # @option opts [Integer, nil] :field_id (nil) field id to store in the schema
       # @return [Node] the added node
       # @raise [ArgumentError] for an unknown unit or a duplicate name
-      def time(name, unit: :micros, utc: true, **opts) = column(name, :time, unit: unit, utc: utc, **opts)
-      # A TIMESTAMP column, stored as INT64.
+      def time(name, unit: :micros, **opts) = column(name, :time, unit: unit, **opts)
+      # A TIME column for wall clock times of day. A Time is written as its time of day in its
+      # own zone, without converting it to UTC. Millis are stored as INT32, micros and nanos as INT64.
       #
       # @param name [String, Symbol] field name
       # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
-      # @param utc [Boolean] the isAdjustedToUTC flag: true for instants, false for local date-times
       # @param opts [Hash{Symbol => Object}] options of #column
       # @option opts [Boolean] :null (true) whether the field is nullable
       # @option opts [Integer, nil] :field_id (nil) field id to store in the schema
       # @return [Node] the added node
       # @raise [ArgumentError] for an unknown unit or a duplicate name
-      def timestamp(name, unit: :micros, utc: true, **opts) = column(name, :timestamp, unit: unit, utc: utc, **opts)
+      def local_time(name, unit: :micros, **opts) = column(name, :local_time, unit: unit, **opts)
+      # A TIMESTAMP column, stored as INT64. A Time in any zone is converted to UTC when written,
+      # so the column holds instants.
+      #
+      # @param name [String, Symbol] field name
+      # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
+      # @param opts [Hash{Symbol => Object}] options of #column
+      # @option opts [Boolean] :null (true) whether the field is nullable
+      # @option opts [Integer, nil] :field_id (nil) field id to store in the schema
+      # @return [Node] the added node
+      # @raise [ArgumentError] for an unknown unit or a duplicate name
+      def timestamp(name, unit: :micros, **opts) = column(name, :timestamp, unit: unit, **opts)
+      # A TIMESTAMP column for wall clock date-times, stored as INT64. A Time is written as its
+      # date and time in its own zone, without converting it to UTC, and no zone is stored.
+      #
+      # @param name [String, Symbol] field name
+      # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
+      # @param opts [Hash{Symbol => Object}] options of #column
+      # @option opts [Boolean] :null (true) whether the field is nullable
+      # @option opts [Integer, nil] :field_id (nil) field id to store in the schema
+      # @return [Node] the added node
+      # @raise [ArgumentError] for an unknown unit or a duplicate name
+      def local_timestamp(name, unit: :micros, **opts) = column(name, :local_timestamp, unit: unit, **opts)
       # A DECIMAL column. Up to 9 digits are stored as INT32, up to 18 as INT64, more as a
       # FIXED_LEN_BYTE_ARRAY of the minimal width, unless +physical:+ says otherwise.
       #
@@ -734,7 +814,6 @@ module Herringbone
       # @option type_opts [Integer] :precision decimal precision
       # @option type_opts [Integer] :scale decimal scale
       # @option type_opts [Symbol] :unit time or timestamp unit
-      # @option type_opts [Boolean] :utc time or timestamp UTC adjustment
       # @option type_opts [Integer] :length FIXED_LEN_BYTE_ARRAY width
       # @yield [list] declares the struct's fields for a +:struct+ element, or exactly one field
       #   (renamed to "element") when +type+ is nil
@@ -773,7 +852,6 @@ module Herringbone
       # @option type_opts [Integer] :precision decimal precision
       # @option type_opts [Integer] :scale decimal scale
       # @option type_opts [Symbol] :unit time or timestamp unit
-      # @option type_opts [Boolean] :utc time or timestamp UTC adjustment
       # @option type_opts [Integer] :length FIXED_LEN_BYTE_ARRAY width
       # @yield [map] declares the struct's fields for a +:struct+ value, or exactly one field
       #   (renamed to "value") when +value_type+ is nil
@@ -795,8 +873,8 @@ module Herringbone
       # Generic column declaration: s.column :name, :int32, null: false
       #
       # @param name [String, Symbol] field name
-      # @param type [Symbol, String] DSL type: one of PRIMITIVES, or +:time+, +:timestamp+,
-      #   +:decimal+, +:fixed+, +:enum+ with their options
+      # @param type [Symbol, String] DSL type: one of PRIMITIVES, or +:time+, +:local_time+,
+      #   +:timestamp+, +:local_timestamp+, +:decimal+, +:fixed+, +:enum+ with their options
       # @param null [Boolean] whether the field is nullable (optional rather than required)
       # @param field_id [Integer, nil] field id to store in the schema
       # @param opts [Hash{Symbol => Object}] type options
@@ -804,7 +882,6 @@ module Herringbone
       # @option opts [Integer] :scale (0) decimal scale
       # @option opts [Symbol] :physical decimal storage: +:int32+, +:int64+, +:binary+ or +:fixed+
       # @option opts [Symbol] :unit (:micros) time or timestamp unit
-      # @option opts [Boolean] :utc (true) time or timestamp UTC adjustment
       # @option opts [Integer] :length FIXED_LEN_BYTE_ARRAY width (required for +:fixed+)
       # @option opts [Array<String>, Hash] :values allowed values, see #enum
       # @option opts [Boolean] :parquet_enum (false) ENUM instead of STRING annotation, see #enum
@@ -908,8 +985,14 @@ module Herringbone
       #   and the rest go to Types.physical_attributes
       # @option opts [Array<String>, Hash] :values allowed values of a string/enum column
       # @return [Node] leaf node, not added to #nodes
-      # @raise [ArgumentError] for an unknown type or invalid type options
+      # @raise [ArgumentError] for an unknown type or invalid type options, or for +utc:+, which
+      #   the local_time and local_timestamp types replace
       def leaf_node(name, type, repetition, opts)
+        if opts.key?(:utc)
+          raise ArgumentError, "#{name}: the utc: option is gone. #{type} columns always convert Times to UTC when " \
+            "writing. To keep the wall clock time as given, without converting it, declare a local_#{type}, " \
+            "e.g. s.local_#{type} :#{name} or s.column :#{name}, :local_#{type}"
+        end
         opts = opts.dup
         values = opts.delete(:values)
         attrs = Types.physical_attributes(type.to_sym, **opts)

data/lib/herringbone/types.rb

@@ -83,13 +83,15 @@ module Herringbone
     end
 
     # Physical attributes (type, logical type etc.) for a DSL type name
+    #
+    # +:time+ and +:timestamp+ hold values converted to UTC and are flagged isAdjustedToUTC;
+    # +:local_time+ and +:local_timestamp+ hold wall clock values, are not flagged and get no
+    # legacy converted type.
     # @param type [Symbol, String] column type name, e.g. +:string+, +:int64+, +:decimal+, +:timestamp+
     # @param opts [Hash{Symbol => Object}] type-specific options
     # @option opts [Boolean] :parquet_enum for +:enum+, annotate as ENUM instead of STRING
     # @option opts [Integer] :length byte length for +:fixed+ (required)
-    # @option opts [Symbol] :unit (:micros) for +:time+ and +:timestamp+, see {time_unit}
-    # @option opts [Boolean] :utc (true) for +:time+ and +:timestamp+, sets +is_adjusted_to_utc+;
-    #   the legacy converted type is only written when true
+    # @option opts [Symbol] :unit (:micros) for the time and timestamp types, see {time_unit}
     # @option opts [Integer] :precision number of decimal digits for +:decimal+ (required)
     # @option opts [Integer] :scale (0) digits after the decimal point for +:decimal+
     # @option opts [Symbol] :physical for +:decimal+, force +:int32+, +:int64+, +:binary+ or
@@ -127,16 +129,18 @@ module Herringbone
       when :int96 then {type: T::INT96}
       when :fixed
         {type: T::FIXED_LEN_BYTE_ARRAY, type_length: Integer(opts.fetch(:length))}
-      when :time
+      when :time, :local_time
+        utc = type.to_sym == :time
         unit = time_unit(opts.fetch(:unit, :micros))
         converted = {millis: C::TIME_MILLIS, micros: C::TIME_MICROS}[unit.to_sym]
-        {type: unit.millis ? T::INT32 : T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
-         logical_type: lt(time: Format::TimeType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit))}
-      when :timestamp
+        {type: unit.millis ? T::INT32 : T::INT64, converted_type: utc ? converted : nil,
+         logical_type: lt(time: Format::TimeType.new(is_adjusted_to_utc: utc, unit: unit))}
+      when :timestamp, :local_timestamp
+        utc = type.to_sym == :timestamp
         unit = time_unit(opts.fetch(:unit, :micros))
         converted = {millis: C::TIMESTAMP_MILLIS, micros: C::TIMESTAMP_MICROS}[unit.to_sym]
-        {type: T::INT64, converted_type: opts.fetch(:utc, true) ? converted : nil,
-         logical_type: lt(timestamp: Format::TimestampType.new(is_adjusted_to_utc: opts.fetch(:utc, true), unit: unit))}
+        {type: T::INT64, converted_type: utc ? converted : nil,
+         logical_type: lt(timestamp: Format::TimestampType.new(is_adjusted_to_utc: utc, unit: unit))}
       when :decimal
         precision = Integer(opts.fetch(:precision))
         scale = Integer(opts.fetch(:scale, 0))
@@ -337,7 +341,8 @@ module Herringbone
     #   date:      Date, Time/DateTime (its calendar date), ISO-8601 String, Integer days since epoch
     #   timestamp: Time, DateTime, ActiveSupport::TimeWithZone, Date (midnight UTC), ISO-8601 String,
     #              Integer in the column's unit
-    #   time:      Time (its time of day), "HH:MM[:SS[.fraction]]" String, Integer in the column's unit
+    #   time:      Time (its time of day, in UTC unless the column is local), "HH:MM[:SS[.fraction]]"
+    #              String, Integer in the column's unit
     #   json:      String (used as-is) or any other object (serialized with JSON.generate)
     #   string:    String, Symbol or anything responding to to_s
     #   boolean:   true/false, 1/0, "true"/"false", "t"/"f", "1"/"0", "yes"/"no"
@@ -357,7 +362,7 @@ module Herringbone
       when :timestamp
         return timestamp_writer(a, b)
       when :time
-        return time_writer(a) if type == T::INT32 || type == T::INT64
+        return time_writer(a, b) if type == T::INT32 || type == T::INT64
       when :json
         return ->(v) { v.is_a?(String) ? v : JSON.generate(v) }
       when :string, :enum
@@ -483,9 +488,11 @@ module Herringbone
     # anything responding to +hour+ and +nsec+ (Time, DateTime), and raises ArgumentError or
     # RangeError for anything else or a time past 23:59:59.
     # @param unit [Symbol] +:millis+, +:micros+ or +:nanos+
+    # @param utc [Boolean] true to convert a Time to UTC before taking its time of day, false to
+    #   take the time of day in the Time's own zone
     # @return [Proc] one-argument lambda
     # @raise [KeyError] for an unknown unit
-    def time_writer(unit)
+    def time_writer(unit, utc)
       mult = UNIT_DIVISORS.fetch(unit)
       lambda do |v|
         return v if v.is_a?(Integer)
@@ -496,6 +503,7 @@ module Herringbone
           secs = m[1].to_i * 3600 + m[2].to_i * 60 + m[3].to_i
         else
           raise ArgumentError, "expected a Time, got #{v.class}" unless v.respond_to?(:hour) && v.respond_to?(:nsec)
+          v = v.getutc if utc && v.respond_to?(:getutc)
           nanos = v.nsec
           secs = v.hour * 3600 + v.min * 60 + v.sec
         end

data/lib/herringbone/version.rb

@@ -2,5 +2,5 @@
 
 module Herringbone
   # Gem version, also written to the +created_by+ field of every file's footer
-  VERSION = "0.7.0"
+  VERSION = "0.9.0"
 end

data/lib/herringbone.rb

@@ -110,6 +110,7 @@ module Herringbone
     autoload :LZ4, "#{LIB}/codecs/lz4"
     autoload :LZO, "#{LIB}/codecs/lzo"
     autoload :Snappy, "#{LIB}/codecs/snappy"
+    autoload :Zstd, "#{LIB}/codecs/zstd"
   end
 
   module Encodings

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: herringbone
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 bindir: bin
 cert_chain: []
-date: 2026-10-09 00:00:00.000000000 Z
+date: 2026-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -45,6 +45,7 @@ files:
 - lib/herringbone/codecs/lz4.rb
 - lib/herringbone/codecs/lzo.rb
 - lib/herringbone/codecs/snappy.rb
+- lib/herringbone/codecs/zstd.rb
 - lib/herringbone/combiner.rb
 - lib/herringbone/compression.rb
 - lib/herringbone/encodings/delta.rb