red-arrow 6.0.1 → 9.0.0

data/lib/arrow/field.rb

@@ -17,12 +17,28 @@
 
 module Arrow
   class Field
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          begin
+            new(value)
+          rescue ArgumentError
+            nil
+          end
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :initialize_raw, :initialize
     private :initialize_raw
 
     # Creates a new {Arrow::Field}.
     #
-    # @overload initialize(name, data_type)
+    # @overload initialize(name, data_type, nullable=false)
     #
     #   @param name [String, Symbol] The name of the field.
     #
@@ -34,6 +50,11 @@ module Arrow
     #     See {Arrow::DataType.resolve} how to specify data type
     #     description.
     #
+    #   @param nullable [Boolean, nil] (false) Whether the field may contain
+    #     zero or more nulls.
+    #
+    #     `nil` is processed as `true` (null may be contained).
+    #
     #   @example Create a field with {Arrow::DataType}s
     #     Arrow::Field.new("visible", Arrow::BooleanDataType.new)
     #
@@ -75,6 +96,11 @@ module Arrow
     #     See {Arrow::DataType.resolve} how to specify data type
     #     description.
     #
+    #   @option description [Boolean, nil] :nullable Whether the field
+    #     may contain zero or more nulls.
+    #
+    #     `nil` is processed as `true` (null may be contained).
+    #
     #   @example Create a field with {Arrow::DataType}s
     #     Arrow::Field.new(name: "visible",
     #                      data_type: Arrow::BooleanDataType.new)
@@ -92,6 +118,7 @@ module Arrow
         name = nil
         data_type = nil
         data_type_description = {}
+        nullable = nil
         description.each do |key, value|
           key = key.to_sym
           case key
@@ -99,20 +126,34 @@ module Arrow
             name = value
           when :data_type
             data_type = DataType.resolve(value)
+          when :nullable
+            nullable = value
           else
             data_type_description[key] = value
           end
         end
         data_type ||= DataType.resolve(data_type_description)
       when 2
+        name = args[0]
+        data_type = args[1]
+        if data_type.is_a?(Hash)
+          data_type = data_type.dup
+          nullable = data_type.delete(:nullable)
+        else
+          nullable = nil
+        end
+        data_type = DataType.resolve(data_type)
+      when 3
         name = args[0]
         data_type = DataType.resolve(args[1])
+        nullable = args[2]
       else
-        message = "wrong number of arguments (given #{n_args}, expected 1..2)"
+        message = "wrong number of arguments (given #{n_args}, expected 1..3)"
         raise ArgumentError, message
       end
+      nullable = true if nullable.nil?
 
-      initialize_raw(name, data_type)
+      initialize_raw(name, data_type, nullable)
     end
   end
 end

data/lib/arrow/function.rb

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Function
+    alias_method :execute_raw, :execute
+    def execute(args, options=nil, context=nil)
+      options = resolve_options(options)
+      execute_raw(args, options, context)
+    end
+    alias_method :call, :execute
+
+    private
+    def resolve_options(options)
+      return nil if options.nil?
+      return options if options.is_a?(FunctionOptions)
+
+      arrow_options_class = options_type&.to_class
+      if arrow_options_class
+        if arrow_options_class.respond_to?(:try_convert)
+          arrow_options = arrow_options_class.try_convert(options)
+          return arrow_options if arrow_options
+        end
+        arrow_options = (default_options || arrow_options_class.new)
+      else
+        arrow_options = default_options
+      end
+      return arrow_options if arrow_options.nil?
+
+      options.each do |key, value|
+        setter = :"#{key}="
+        next unless arrow_options.respond_to?(setter)
+        arrow_options.__send__(setter, value)
+      end
+      arrow_options
+    end
+  end
+end

data/lib/arrow/list-data-type.rb

@@ -107,12 +107,7 @@ module Arrow
         description = arg
         arg = description[:field]
       end
-      if arg.is_a?(Hash)
-        field_description = arg
-        Field.new(field_description)
-      else
-        arg
-      end
+      arg
     end
   end
 end

data/lib/arrow/loader.rb

@@ -30,9 +30,11 @@ module Arrow
       require_libraries
       require_extension_library
       gc_guard
+      self.class.start_callback_dispatch_thread
     end
 
     def require_libraries
+      require "arrow/array-computable"
       require "arrow/column-containable"
       require "arrow/field-containable"
       require "arrow/generic-filterable"
@@ -58,6 +60,7 @@ module Arrow
       require "arrow/date64-array"
       require "arrow/date64-array-builder"
       require "arrow/datum"
+      require "arrow/day-time-interval-array-builder"
       require "arrow/decimal128"
       require "arrow/decimal128-array"
       require "arrow/decimal128-array-builder"
@@ -76,12 +79,14 @@ module Arrow
       require "arrow/file-system"
       require "arrow/fixed-size-binary-array"
       require "arrow/fixed-size-binary-array-builder"
+      require "arrow/function"
       require "arrow/group"
       require "arrow/list-array-builder"
       require "arrow/list-data-type"
       require "arrow/map-array"
       require "arrow/map-array-builder"
       require "arrow/map-data-type"
+      require "arrow/month-day-nano-interval-array-builder"
       require "arrow/null-array"
       require "arrow/null-array-builder"
       require "arrow/path-extension"
@@ -93,6 +98,7 @@ module Arrow
       require "arrow/record-batch-reader"
       require "arrow/record-batch-stream-reader"
       require "arrow/rolling-window"
+      require "arrow/s3-global-options"
       require "arrow/scalar"
       require "arrow/schema"
       require "arrow/slicer"
@@ -101,6 +107,7 @@ module Arrow
       require "arrow/source-node-options"
       require "arrow/sparse-union-data-type"
       require "arrow/string-dictionary-array-builder"
+      require "arrow/string-array-builder"
       require "arrow/struct-array"
       require "arrow/struct-array-builder"
       require "arrow/struct-data-type"
@@ -113,6 +120,7 @@ module Arrow
       require "arrow/table-saver"
       require "arrow/tensor"
       require "arrow/time"
+      require "arrow/time-unit"
       require "arrow/time32-array"
       require "arrow/time32-array-builder"
       require "arrow/time32-data-type"
@@ -212,5 +220,16 @@ module Arrow
         super
       end
     end
+
+    def prepare_function_info_lock_gvl(function_info, klass)
+      super
+      case klass.name
+      when "Arrow::RecordBatchFileReader"
+        case function_info.name
+        when "new"
+          function_info.lock_gvl_default = false
+        end
+      end
+    end
   end
 end

data/lib/arrow/month-day-nano-interval-array-builder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class MonthDayNanoIntervalArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      if value.is_a?(Hash)
+        Arrow::MonthDayNano.new(value[:month], value[:day], value[:nanosecond])
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/s3-global-options.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class S3GlobalOptions
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          options = new
+          value.each do |k, v|
+            setter = :"#{k}="
+            return unless options.respond_to?(setter)
+            options.__send__(setter, v)
+          end
+          options
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/sort-key.rb

@@ -29,25 +29,26 @@ module Arrow
       #
       #   @return [Arrow::SortKey] The given sort key itself.
       #
-      # @overload resolve(name)
+      # @overload resolve(target)
       #
-      #   Creates a new suitable sort key from column name with
-      #   leading order mark. See {#initialize} for details about
+      #   Creates a new suitable sort key from column name or dot path
+      #   with leading order mark. See {#initialize} for details about
       #   order mark.
       #
       #   @return [Arrow::SortKey] A new suitable sort key.
       #
-      # @overload resolve(name, order)
+      # @overload resolve(target, order)
       #
-      #   Creates a new suitable sort key from column name without
-      #   leading order mark and order. See {#initialize} for details.
+      #   Creates a new suitable sort key from column name or dot path
+      #   without leading order mark and order. See {#initialize} for
+      #   details.
       #
       #   @return [Arrow::SortKey] A new suitable sort key.
       #
       # @since 4.0.0
-      def resolve(name, order=nil)
-        return name if name.is_a?(self)
-        new(name, order)
+      def resolve(target, order=nil)
+        return target if target.is_a?(self)
+        new(target, order)
       end
 
       # @api private
@@ -65,47 +66,49 @@ module Arrow
     private :initialize_raw
     # Creates a new {Arrow::SortKey}.
     #
-    # @overload initialize(name)
+    # @overload initialize(target)
     #
-    #   @param name [Symbol, String] The name of the sort column.
+    #   @param target [Symbol, String] The name or dot path of the
+    #     sort column.
     #
-    #     If `name` is a String, the first character may be processed
-    #     as the "leading order mark". If the first character is `"+"`
-    #     or `"-"`, they are processed as a leading order mark. If the
-    #     first character is processed as a leading order mark, the
-    #     first character is removed from sort column name and
-    #     corresponding order is used. `"+"` uses ascending order and
-    #     `"-"` uses ascending order.
+    #     If `target` is a String, the first character may be
+    #     processed as the "leading order mark". If the first
+    #     character is `"+"` or `"-"`, they are processed as a leading
+    #     order mark. If the first character is processed as a leading
+    #     order mark, the first character is removed from sort column
+    #     target and corresponding order is used. `"+"` uses ascending
+    #     order and `"-"` uses ascending order.
     #
-    #     If `name` is not a String nor `name` doesn't start with the
-    #     leading order mark, sort column name is `name` as-is and
+    #     If `target` is not a String nor `target` doesn't start with the
+    #     leading order mark, sort column target is `target` as-is and
     #     ascending order is used.
     #
     #   @example String without the leading order mark
     #     key = Arrow::SortKey.new("count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
     #   @example String with the "+" leading order mark
     #     key = Arrow::SortKey.new("+count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
     #   @example String with the "-" leading order mark
     #     key = Arrow::SortKey.new("-count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Symbol that starts with "-"
     #     key = Arrow::SortKey.new(:"-count")
-    #     key.name  # => "-count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "-count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
-    # @overload initialize(name, order)
+    # @overload initialize(target, order)
     #
-    #   @param name [Symbol, String] The name of the sort column.
+    #   @param target [Symbol, String] The name or dot path of the
+    #     sort column.
     #
-    #     No leading order mark processing. The given `name` is used
+    #     No leading order mark processing. The given `target` is used
     #     as-is.
     #
     #   @param order [Symbol, String, Arrow::SortOrder] How to order
@@ -117,29 +120,29 @@ module Arrow
     #
     #   @example No leading order mark processing
     #     key = Arrow::SortKey.new("-count", :ascending)
-    #     key.name  # => "-count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "-count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
-    #   @example Order by abbreviated name with Symbol
+    #   @example Order by abbreviated target with Symbol
     #     key = Arrow::SortKey.new("count", :desc)
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Order by String
     #     key = Arrow::SortKey.new("count", "descending")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Order by Arrow::SortOrder
     #     key = Arrow::SortKey.new("count", Arrow::SortOrder::DESCENDING)
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     # @since 4.0.0
-    def initialize(name, order=nil)
-      name, order = normalize_name(name, order)
+    def initialize(target, order=nil)
+      target, order = normalize_target(target, order)
       order = normalize_order(order) || :ascending
-      initialize_raw(name, order)
+      initialize_raw(target, order)
     end
 
     # @return [String] The string representation of this sort key. You
@@ -154,28 +157,31 @@ module Arrow
     # @since 4.0.0
     def to_s
       if order == SortOrder::ASCENDING
-        "+#{name}"
+        "+#{target}"
       else
-        "-#{name}"
+        "-#{target}"
       end
     end
 
+    # For backward compatibility
+    alias_method :name, :target
+
     private
-    def normalize_name(name, order)
-      case name
+    def normalize_target(target, order)
+      case target
       when Symbol
-        return name.to_s, order
+        return target.to_s, order
       when String
-        return name, order if order
-        if name.start_with?("-")
-          return name[1..-1], order || :descending
-        elsif name.start_with?("+")
-          return name[1..-1], order || :ascending
+        return target, order if order
+        if target.start_with?("-")
+          return target[1..-1], order || :descending
+        elsif target.start_with?("+")
+          return target[1..-1], order || :ascending
         else
-          return name, order
+          return target, order
         end
       else
-        return name, order
+        return target, order
       end
     end
 

data/lib/arrow/sort-options.rb

@@ -78,20 +78,20 @@ module Arrow
     #     options.add_sort_key(Arrow::SortKey.new(:price, :descending))
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
-    # @overload add_sort_key(name)
+    # @overload add_sort_key(target)
     #
-    #   @param name [Symbol, String] The sort key name to be
-    #     added. See also {Arrow::SortKey#initialize} for the leading
-    #     order mark for String name.
+    #   @param target [Symbol, String] The sort key name or dot path
+    #     to be added. See also {Arrow::SortKey#initialize} for the
+    #     leading order mark for `String` target.
     #
     #   @example Add a key to sort by "price" column in descending order
     #     options = Arrow::SortOptions.new
     #     options.add_sort_key("-price")
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
-    # @overload add_sort_key(name, order)
+    # @overload add_sort_key(target, order)
     #
-    #   @param name [Symbol, String] The sort key name.
+    #   @param target [Symbol, String] The sort key name or dot path.
     #
     #   @param order [Symbol, String, Arrow::SortOrder] The sort
     #     order. See {Arrow::SortKey#initialize} for details.
@@ -102,8 +102,8 @@ module Arrow
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
     # @since 4.0.0
-    def add_sort_key(name, order=nil)
-      add_sort_key_raw(SortKey.resolve(name, order))
+    def add_sort_key(target, order=nil)
+      add_sort_key_raw(SortKey.resolve(target, order))
     end
   end
 end

data/lib/arrow/string-array-builder.rb

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class StringArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      case value
+      when GLib::Bytes, String
+        value
+      else
+        value.to_s
+      end
+    end
+  end
+end