red-arrow 8.0.0 → 9.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f43d3875f2e48876cd47533124a9fcf9559e263e63e3954c65fedd36b4e59744
-  data.tar.gz: 2548f661536d3288f4ad8bb25d6587a78ee51d810a85a7d533ccdaf595439a12
+  metadata.gz: 23d181c0c2d49f1479ddae27c02427d75b72f66a31c097e1f0ead363422170ed
+  data.tar.gz: e2c5e259b177bb136d7f492e46ec3411b5f05d190618cec6736494315fc64efb
 SHA512:
-  metadata.gz: caeffbc164f14db7b056263f1ecd449afe2feec3ccbf7b883c147d415199c9ddc6c20398b00c34884248c103fbb7c55aa225719d8b8e75ac7f49337ee89d557e
-  data.tar.gz: ae5a793e06b63bd80e3499f794193f47f72ff7510a558d2d0e61314068d5bd3a1986e5083bda749b00b73a4615901beb320521eeb360576b93fcb9a1e9d9510a
+  metadata.gz: a4d7673b6457b4f288d9fc1e0bcc8b9adad239f21dbe722cbb412e81e51d117e8661027731bd4918be19c9ed6c6978f21613ebdd31fbe21271d2175f3d416655
+  data.tar.gz: f300245a89afeb3c8d5e56db26fe866ae666a429429c26155591b430bc47f9d7abe987e35fd0507133b9e5011f78381b6e46797b1072556786fb8fb54e19e9fe

data/ext/arrow/converters.hpp

@@ -773,6 +773,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)

data/ext/arrow/extconf.rb

@@ -45,9 +45,10 @@ unless required_pkg_config_package([
                                      Arrow::Version::MICRO,
                                    ],
                                    debian: "libarrow-dev",
-                                   redhat: "arrow-devel",
+                                   fedora: "libarrow-devel",
                                    homebrew: "apache-arrow",
-                                   msys2: "arrow")
+                                   msys2: "arrow",
+                                   redhat: "arrow-devel")
   exit(false)
 end
 
@@ -58,9 +59,10 @@ unless required_pkg_config_package([
                                      Arrow::Version::MICRO,
                                    ],
                                    debian: "libarrow-glib-dev",
-                                   redhat: "arrow-glib-devel",
+                                   fedora: "libarrow-glib-devel",
                                    homebrew: "apache-arrow-glib",
-                                   msys2: "arrow")
+                                   msys2: "arrow",
+                                   redhat: "arrow-glib-devel")
   exit(false)
 end
 

data/lib/arrow/array-builder.rb

@@ -33,6 +33,11 @@ module Arrow
         end
         if builder_info
           builder = builder_info[:builder]
+          if builder.nil? and builder_info[:builder_type]
+            builder = create_builder(builder_info)
+          end
+        end
+        if builder
           builder.build(values)
         else
           Arrow::StringArray.new(values)
@@ -121,15 +126,28 @@ module Arrow
             detected: true,
           }
         when BigDecimal
-          if value.to_arrow.is_a?(Decimal128)
-            {
-              builder: Decimal128ArrayBuilder.new,
-            }
-          else
+          builder_info ||= {}
+          if builder_info[:builder] or value.nan? or value.infinite?
             {
-              builder: Decimal256ArrayBuilder.new,
+              builder: StringArrayBuilder.new,
               detected: true,
             }
+          else
+            precision = [builder_info[:precision] || 0, value.precision].max
+            scale = [builder_info[:scale] || 0, value.scale].max
+            if precision <= Decimal128DataType::MAX_PRECISION
+              {
+                builder_type: :decimal128,
+                precision: precision,
+                scale: scale,
+              }
+            else
+              {
+                builder_type: :decimal256,
+                precision: precision,
+                scale: scale,
+              }
+            end
           end
         when ::Array
           sub_builder_info = nil
@@ -154,6 +172,22 @@ module Arrow
           }
         end
       end
+
+      def create_builder(builder_info)
+        builder_type = builder_info[:builder_type]
+        case builder_type
+        when :decimal128
+          data_type = Decimal128DataType.new(builder_info[:precision],
+                                             builder_info[:scale])
+          Decimal128ArrayBuilder.new(data_type)
+        when :decimal256
+          data_type = Decimal256DataType.new(builder_info[:precision],
+                                             builder_info[:scale])
+          Decimal256ArrayBuilder.new(data_type)
+        else
+          nil
+        end
+      end
     end
 
     def build(values)

data/lib/arrow/array-computable.rb

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module ArrayComputable
+    def min(options: nil)
+      compute("min", options: options).value
+    end
+
+    def max(options: nil)
+      compute("max", options: options).value
+    end
+
+    def uniq
+      unique.values
+    end
+
+    private
+    def compute(name, options: nil)
+      Function.find(name).execute([self], options).value
+    end
+  end
+end

data/lib/arrow/array.rb

@@ -18,6 +18,8 @@
 module Arrow
   class Array
     include Enumerable
+
+    include ArrayComputable
     include GenericFilterable
     include GenericTakeable
 
@@ -34,6 +36,20 @@ module Arrow
         return nil unless const_defined?(builder_class_name)
         const_get(builder_class_name)
       end
+
+      # @api private
+      def try_convert(value)
+        case value
+        when ::Array
+          begin
+            new(value)
+          rescue ArgumentError
+            nil
+          end
+        else
+          nil
+        end
+      end
     end
 
     # @param i [Integer]

data/lib/arrow/chunked-array.rb

@@ -18,6 +18,8 @@
 module Arrow
   class ChunkedArray
     include Enumerable
+
+    include ArrayComputable
     include GenericFilterable
     include GenericTakeable
 
@@ -87,5 +89,24 @@ module Arrow
         first_chunk.class.new(to_a)
       end
     end
+
+    def count(options: nil)
+      compute("count", options: options).value
+    end
+
+    def sum(options: nil)
+      compute("sum", options: options).value
+    end
+
+    def unique
+      compute("unique")
+    end
+
+    def cast(target_data_type, options: nil)
+      casted_chunks = chunks.collect do |chunk|
+        chunk.cast(target_data_type, options)
+      end
+      self.class.new(casted_chunks)
+    end
   end
 end

data/lib/arrow/column.rb

@@ -72,5 +72,33 @@ module Arrow
         @field == other.field and
         @data == other.data
     end
+
+    def count(options: nil)
+      @data.count(options: options)
+    end
+
+    def sum(options: nil)
+      @data.sum(options: options)
+    end
+
+    def min(options: nil)
+      @data.min(options: options)
+    end
+
+    def max(options: nil)
+      @data.max(options: options)
+    end
+
+    def unique
+      @data.unique
+    end
+
+    def uniq
+      @data.uniq
+    end
+
+    def cast(target_data_type, options: nil)
+      @data.cast(target_data_type, options: options)
+    end
   end
 end

data/lib/arrow/data-type.rb

@@ -110,7 +110,7 @@ module Arrow
               description[key] = value
             end
           end
-          if type.nil?
+          if type.nil? and self == DataType
             message =
               "data type description must have :type value: #{data_type.inspect}"
             raise ArgumentError, message
@@ -152,6 +152,7 @@ module Arrow
 
       private
       def resolve_class(data_type)
+        return self if data_type.nil?
         components = data_type.to_s.split("_").collect(&:capitalize)
         data_type_name = components.join.gsub(/\AUint/, "UInt")
         data_type_class_name = "#{data_type_name}DataType"

data/lib/arrow/decimal128-array-builder.rb

@@ -42,16 +42,26 @@ module Arrow
     end
 
     private
+    def precision
+      @precision ||= value_data_type.precision
+    end
+
+    def scale
+      @scale ||= value_data_type.scale
+    end
+
     def normalize_value(value)
       case value
-      when String
-        Decimal128.new(value)
-      when Float
-        Decimal128.new(value.to_s)
       when BigDecimal
-        Decimal128.new(value.to_s)
+        if value.nan? or value.infinite?
+          message = "can't use #{value} as an Arrow::Decimal128Array value"
+          raise FloatDomainError, message
+        end
+        integer, decimal = value.to_s("f").split(".", 2)
+        decimal = decimal[0, scale].ljust(scale, "0")
+        Decimal128.new("#{integer}.#{decimal}")
       else
-        value
+        Decimal128.try_convert(value) || value
       end
     end
   end

data/lib/arrow/decimal128.rb

@@ -17,6 +17,20 @@
 
 module Arrow
   class Decimal128
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        when Float
+          new(value.to_s)
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :to_s_raw, :to_s
 
     # @overload to_s

data/lib/arrow/decimal256-array-builder.rb

@@ -45,16 +45,26 @@ module Arrow
     end
 
     private
+    def precision
+      @precision ||= value_data_type.precision
+    end
+
+    def scale
+      @scale ||= value_data_type.scale
+    end
+
     def normalize_value(value)
       case value
-      when String
-        Decimal256.new(value)
-      when Float
-        Decimal256.new(value.to_s)
       when BigDecimal
-        Decimal256.new(value.to_s)
+        if value.nan? or value.infinite?
+          message = "can't use #{value} as an Arrow::Decimal256Array value"
+          raise FloatDomainError, message
+        end
+        integer, decimal = value.to_s("f").split(".", 2)
+        decimal = decimal[0, scale].ljust(scale, "0")
+        Decimal256.new("#{integer}.#{decimal}")
       else
-        value
+        Decimal256.try_convert(value) || value
       end
     end
   end

data/lib/arrow/decimal256.rb

@@ -17,6 +17,20 @@
 
 module Arrow
   class Decimal256
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        when Float
+          new(value.to_s)
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :to_s_raw, :to_s
 
     # @overload to_s

data/lib/arrow/field.rb

@@ -17,12 +17,28 @@
 
 module Arrow
   class Field
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          begin
+            new(value)
+          rescue ArgumentError
+            nil
+          end
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :initialize_raw, :initialize
     private :initialize_raw
 
     # Creates a new {Arrow::Field}.
     #
-    # @overload initialize(name, data_type)
+    # @overload initialize(name, data_type, nullable=false)
     #
     #   @param name [String, Symbol] The name of the field.
     #
@@ -34,6 +50,11 @@ module Arrow
     #     See {Arrow::DataType.resolve} how to specify data type
     #     description.
     #
+    #   @param nullable [Boolean, nil] (false) Whether the field may contain
+    #     zero or more nulls.
+    #
+    #     `nil` is processed as `true` (null may be contained).
+    #
     #   @example Create a field with {Arrow::DataType}s
     #     Arrow::Field.new("visible", Arrow::BooleanDataType.new)
     #
@@ -75,6 +96,11 @@ module Arrow
     #     See {Arrow::DataType.resolve} how to specify data type
     #     description.
     #
+    #   @option description [Boolean, nil] :nullable Whether the field
+    #     may contain zero or more nulls.
+    #
+    #     `nil` is processed as `true` (null may be contained).
+    #
     #   @example Create a field with {Arrow::DataType}s
     #     Arrow::Field.new(name: "visible",
     #                      data_type: Arrow::BooleanDataType.new)
@@ -92,6 +118,7 @@ module Arrow
         name = nil
         data_type = nil
         data_type_description = {}
+        nullable = nil
         description.each do |key, value|
           key = key.to_sym
           case key
@@ -99,20 +126,34 @@ module Arrow
             name = value
           when :data_type
             data_type = DataType.resolve(value)
+          when :nullable
+            nullable = value
           else
             data_type_description[key] = value
           end
         end
         data_type ||= DataType.resolve(data_type_description)
       when 2
+        name = args[0]
+        data_type = args[1]
+        if data_type.is_a?(Hash)
+          data_type = data_type.dup
+          nullable = data_type.delete(:nullable)
+        else
+          nullable = nil
+        end
+        data_type = DataType.resolve(data_type)
+      when 3
         name = args[0]
         data_type = DataType.resolve(args[1])
+        nullable = args[2]
       else
-        message = "wrong number of arguments (given #{n_args}, expected 1..2)"
+        message = "wrong number of arguments (given #{n_args}, expected 1..3)"
         raise ArgumentError, message
       end
+      nullable = true if nullable.nil?
 
-      initialize_raw(name, data_type)
+      initialize_raw(name, data_type, nullable)
     end
   end
 end

data/lib/arrow/list-data-type.rb

@@ -107,12 +107,7 @@ module Arrow
         description = arg
         arg = description[:field]
       end
-      if arg.is_a?(Hash)
-        field_description = arg
-        Field.new(field_description)
-      else
-        arg
-      end
+      arg
     end
   end
 end

data/lib/arrow/loader.rb

@@ -34,6 +34,7 @@ module Arrow
     end
 
     def require_libraries
+      require "arrow/array-computable"
       require "arrow/column-containable"
       require "arrow/field-containable"
       require "arrow/generic-filterable"
@@ -106,6 +107,7 @@ module Arrow
       require "arrow/source-node-options"
       require "arrow/sparse-union-data-type"
       require "arrow/string-dictionary-array-builder"
+      require "arrow/string-array-builder"
       require "arrow/struct-array"
       require "arrow/struct-array-builder"
       require "arrow/struct-data-type"
@@ -118,6 +120,7 @@ module Arrow
       require "arrow/table-saver"
       require "arrow/tensor"
       require "arrow/time"
+      require "arrow/time-unit"
       require "arrow/time32-array"
       require "arrow/time32-array-builder"
       require "arrow/time32-data-type"

data/lib/arrow/string-array-builder.rb

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class StringArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      case value
+      when GLib::Bytes, String
+        value
+      else
+        value.to_s
+      end
+    end
+  end
+end

data/lib/arrow/time-unit.rb

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class TimeUnit
+    class << self
+      # @api private
+      def try_convert(value)
+        if value.is_a?(Hash) and value.size == 1 and value[:unit]
+          super(value[:unit])
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/arrow/time32-array-builder.rb

@@ -18,24 +18,12 @@
 module Arrow
   class Time32ArrayBuilder
     class << self
-      def build(unit_or_data_type, values)
-        builder = new(unit_or_data_type)
+      def build(data_type, values)
+        builder = new(data_type)
         builder.build(values)
       end
     end
 
-    alias_method :initialize_raw, :initialize
-    def initialize(unit_or_data_type)
-      case unit_or_data_type
-      when DataType
-        data_type = unit_or_data_type
-      else
-        unit = unit_or_data_type
-        data_type = Time32DataType.new(unit)
-      end
-      initialize_raw(data_type)
-    end
-
     def unit
       @unit ||= value_data_type.unit
     end

data/lib/arrow/time32-data-type.rb

@@ -17,45 +17,16 @@
 
 module Arrow
   class Time32DataType
-    alias_method :initialize_raw, :initialize
-    private :initialize_raw
-
-    # Creates a new {Arrow::Time32DataType}.
-    #
-    # @overload initialize(unit)
-    #
-    #   @param unit [Arrow::TimeUnit, Symbol] The unit of the
-    #     time32 data type.
-    #
-    #     The unit must be second or millisecond.
-    #
-    #   @example Create a time32 data type with Arrow::TimeUnit
-    #     Arrow::Time32DataType.new(Arrow::TimeUnit::MILLI)
-    #
-    #   @example Create a time32 data type with Symbol
-    #     Arrow::Time32DataType.new(:milli)
-    #
-    # @overload initialize(description)
-    #
-    #   @param description [Hash] The description of the time32 data
-    #     type. It must have `:unit` value.
-    #
-    #   @option description [Arrow::TimeUnit, Symbol] :unit The unit of
-    #     the time32 data type.
-    #
-    #     The unit must be second or millisecond.
-    #
-    #   @example Create a time32 data type with Arrow::TimeUnit
-    #     Arrow::Time32DataType.new(unit: Arrow::TimeUnit::MILLI)
-    #
-    #   @example Create a time32 data type with Symbol
-    #     Arrow::Time32DataType.new(unit: :milli)
-    def initialize(unit)
-      if unit.is_a?(Hash)
-        description = unit
-        unit = description[:unit]
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Symbol, Arrow::TimeUnit
+          new(value)
+        else
+          super
+        end
       end
-      initialize_raw(unit)
     end
   end
 end