red-arrow 8.0.0 → 10.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f43d3875f2e48876cd47533124a9fcf9559e263e63e3954c65fedd36b4e59744
-  data.tar.gz: 2548f661536d3288f4ad8bb25d6587a78ee51d810a85a7d533ccdaf595439a12
+  metadata.gz: dc23de72fe60f7f6b71d45d4ec9019b0066da0de1aaef7992af984fa3ee14db3
+  data.tar.gz: 5383f405645f394c8af4b35ce537a82a54b207823a1debf755d09ef645bcbe15
 SHA512:
-  metadata.gz: caeffbc164f14db7b056263f1ecd449afe2feec3ccbf7b883c147d415199c9ddc6c20398b00c34884248c103fbb7c55aa225719d8b8e75ac7f49337ee89d557e
-  data.tar.gz: ae5a793e06b63bd80e3499f794193f47f72ff7510a558d2d0e61314068d5bd3a1986e5083bda749b00b73a4615901beb320521eeb360576b93fcb9a1e9d9510a
+  metadata.gz: cf9d0c9274059ab09a91af39573c5e4b1a902ca486038e61aff882441d725592bc4143821ba82607979ff09ea7cc9e5220efda1f12478e28b2cf9a9c89cff2d2
+  data.tar.gz: 4225c79d4eeb6770175624ef0eb4e90e263715a1d0b817bf34898ad179a85325f0d15e8fca6697d94870b6147bb965ab1644170fdae52eab07c37396214e5549

data/ext/arrow/converters.hpp

@@ -773,6 +773,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)

data/ext/arrow/extconf.rb

@@ -38,6 +38,8 @@ checking_for(checking_message("Homebrew")) do
   end
 end
 
+$CXXFLAGS += " -std=c++17 "
+
 unless required_pkg_config_package([
                                      "arrow",
                                      Arrow::Version::MAJOR,
@@ -45,9 +47,10 @@ unless required_pkg_config_package([
                                      Arrow::Version::MICRO,
                                    ],
                                    debian: "libarrow-dev",
-                                   redhat: "arrow-devel",
+                                   fedora: "libarrow-devel",
                                    homebrew: "apache-arrow",
-                                   msys2: "arrow")
+                                   msys2: "arrow",
+                                   redhat: "arrow-devel")
   exit(false)
 end
 
@@ -58,9 +61,10 @@ unless required_pkg_config_package([
                                      Arrow::Version::MICRO,
                                    ],
                                    debian: "libarrow-glib-dev",
-                                   redhat: "arrow-glib-devel",
+                                   fedora: "libarrow-glib-devel",
                                    homebrew: "apache-arrow-glib",
-                                   msys2: "arrow")
+                                   msys2: "arrow",
+                                   redhat: "arrow-glib-devel")
   exit(false)
 end
 

data/lib/arrow/array-builder.rb

@@ -33,6 +33,11 @@ module Arrow
         end
         if builder_info
           builder = builder_info[:builder]
+          if builder.nil? and builder_info[:builder_type]
+            builder = create_builder(builder_info)
+          end
+        end
+        if builder
           builder.build(values)
         else
           Arrow::StringArray.new(values)
@@ -121,15 +126,28 @@ module Arrow
             detected: true,
           }
         when BigDecimal
-          if value.to_arrow.is_a?(Decimal128)
-            {
-              builder: Decimal128ArrayBuilder.new,
-            }
-          else
+          builder_info ||= {}
+          if builder_info[:builder] or value.nan? or value.infinite?
             {
-              builder: Decimal256ArrayBuilder.new,
+              builder: StringArrayBuilder.new,
               detected: true,
             }
+          else
+            precision = [builder_info[:precision] || 0, value.precision].max
+            scale = [builder_info[:scale] || 0, value.scale].max
+            if precision <= Decimal128DataType::MAX_PRECISION
+              {
+                builder_type: :decimal128,
+                precision: precision,
+                scale: scale,
+              }
+            else
+              {
+                builder_type: :decimal256,
+                precision: precision,
+                scale: scale,
+              }
+            end
           end
         when ::Array
           sub_builder_info = nil
@@ -154,6 +172,22 @@ module Arrow
           }
         end
       end
+
+      def create_builder(builder_info)
+        builder_type = builder_info[:builder_type]
+        case builder_type
+        when :decimal128
+          data_type = Decimal128DataType.new(builder_info[:precision],
+                                             builder_info[:scale])
+          Decimal128ArrayBuilder.new(data_type)
+        when :decimal256
+          data_type = Decimal256DataType.new(builder_info[:precision],
+                                             builder_info[:scale])
+          Decimal256ArrayBuilder.new(data_type)
+        else
+          nil
+        end
+      end
     end
 
     def build(values)

data/lib/arrow/array-computable.rb

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module ArrayComputable
+    def min(options: nil)
+      compute("min", options: options).value
+    end
+
+    def max(options: nil)
+      compute("max", options: options).value
+    end
+
+    def uniq
+      unique.values
+    end
+
+    private
+    def compute(name, options: nil)
+      Function.find(name).execute([self], options).value
+    end
+  end
+end

data/lib/arrow/array.rb

@@ -18,6 +18,8 @@
 module Arrow
   class Array
     include Enumerable
+
+    include ArrayComputable
     include GenericFilterable
     include GenericTakeable
 
@@ -34,6 +36,28 @@ module Arrow
         return nil unless const_defined?(builder_class_name)
         const_get(builder_class_name)
       end
+
+      # @api private
+      def try_convert(value)
+        case value
+        when ::Array
+          begin
+            new(value)
+          rescue ArgumentError
+            nil
+          end
+        else
+          if value.respond_to?(:to_arrow_array)
+            begin
+              value.to_arrow_array
+            rescue RangeError
+              nil
+            end
+          else
+            nil
+          end
+        end
+      end
     end
 
     # @param i [Integer]
@@ -87,6 +111,10 @@ module Arrow
       self
     end
 
+    def to_arrow_array
+      self
+    end
+
     alias_method :value_data_type_raw, :value_data_type
     def value_data_type
       @value_data_type ||= value_data_type_raw

data/lib/arrow/chunked-array.rb

@@ -18,6 +18,8 @@
 module Arrow
   class ChunkedArray
     include Enumerable
+
+    include ArrayComputable
     include GenericFilterable
     include GenericTakeable
 
@@ -87,5 +89,24 @@ module Arrow
         first_chunk.class.new(to_a)
       end
     end
+
+    def count(options: nil)
+      compute("count", options: options).value
+    end
+
+    def sum(options: nil)
+      compute("sum", options: options).value
+    end
+
+    def unique
+      compute("unique")
+    end
+
+    def cast(target_data_type, options: nil)
+      casted_chunks = chunks.collect do |chunk|
+        chunk.cast(target_data_type, options)
+      end
+      self.class.new(casted_chunks)
+    end
   end
 end

data/lib/arrow/column.rb

@@ -72,5 +72,33 @@ module Arrow
         @field == other.field and
         @data == other.data
     end
+
+    def count(options: nil)
+      @data.count(options: options)
+    end
+
+    def sum(options: nil)
+      @data.sum(options: options)
+    end
+
+    def min(options: nil)
+      @data.min(options: options)
+    end
+
+    def max(options: nil)
+      @data.max(options: options)
+    end
+
+    def unique
+      @data.unique
+    end
+
+    def uniq
+      @data.uniq
+    end
+
+    def cast(target_data_type, options: nil)
+      @data.cast(target_data_type, options: options)
+    end
   end
 end

data/lib/arrow/data-type.rb

@@ -110,7 +110,7 @@ module Arrow
               description[key] = value
             end
           end
-          if type.nil?
+          if type.nil? and self == DataType
             message =
               "data type description must have :type value: #{data_type.inspect}"
             raise ArgumentError, message
@@ -152,6 +152,7 @@ module Arrow
 
       private
       def resolve_class(data_type)
+        return self if data_type.nil?
         components = data_type.to_s.split("_").collect(&:capitalize)
         data_type_name = components.join.gsub(/\AUint/, "UInt")
         data_type_class_name = "#{data_type_name}DataType"
@@ -187,9 +188,13 @@ module Arrow
       end
     end
 
-    def build_array(values)
+    def array_class
       base_name = self.class.name.gsub(/DataType\z/, "")
-      builder_class = self.class.const_get("#{base_name}ArrayBuilder")
+      ::Arrow.const_get("#{base_name}Array")
+    end
+
+    def build_array(values)
+      builder_class = array_class.builder_class
       args = [values]
       args.unshift(self) unless builder_class.buildable?(args)
       builder_class.build(*args)

data/lib/arrow/decimal128-array-builder.rb

@@ -42,16 +42,26 @@ module Arrow
     end
 
     private
+    def precision
+      @precision ||= value_data_type.precision
+    end
+
+    def scale
+      @scale ||= value_data_type.scale
+    end
+
     def normalize_value(value)
       case value
-      when String
-        Decimal128.new(value)
-      when Float
-        Decimal128.new(value.to_s)
       when BigDecimal
-        Decimal128.new(value.to_s)
+        if value.nan? or value.infinite?
+          message = "can't use #{value} as an Arrow::Decimal128Array value"
+          raise FloatDomainError, message
+        end
+        integer, decimal = value.to_s("f").split(".", 2)
+        decimal = decimal[0, scale].ljust(scale, "0")
+        Decimal128.new("#{integer}.#{decimal}")
       else
-        value
+        Decimal128.try_convert(value) || value
       end
     end
   end

data/lib/arrow/decimal128.rb

@@ -17,6 +17,20 @@
 
 module Arrow
   class Decimal128
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        when Float
+          new(value.to_s)
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :to_s_raw, :to_s
 
     # @overload to_s

data/lib/arrow/decimal256-array-builder.rb

@@ -45,16 +45,26 @@ module Arrow
     end
 
     private
+    def precision
+      @precision ||= value_data_type.precision
+    end
+
+    def scale
+      @scale ||= value_data_type.scale
+    end
+
     def normalize_value(value)
       case value
-      when String
-        Decimal256.new(value)
-      when Float
-        Decimal256.new(value.to_s)
       when BigDecimal
-        Decimal256.new(value.to_s)
+        if value.nan? or value.infinite?
+          message = "can't use #{value} as an Arrow::Decimal256Array value"
+          raise FloatDomainError, message
+        end
+        integer, decimal = value.to_s("f").split(".", 2)
+        decimal = decimal[0, scale].ljust(scale, "0")
+        Decimal256.new("#{integer}.#{decimal}")
       else
-        value
+        Decimal256.try_convert(value) || value
       end
     end
   end

data/lib/arrow/decimal256.rb

@@ -17,6 +17,20 @@
 
 module Arrow
   class Decimal256
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        when Float
+          new(value.to_s)
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :to_s_raw, :to_s
 
     # @overload to_s

data/lib/arrow/field.rb

@@ -17,12 +17,28 @@
 
 module Arrow
   class Field
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          begin
+            new(value)
+          rescue ArgumentError
+            nil
+          end
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :initialize_raw, :initialize
     private :initialize_raw
 
     # Creates a new {Arrow::Field}.
     #
-    # @overload initialize(name, data_type)
+    # @overload initialize(name, data_type, nullable=false)
     #
     #   @param name [String, Symbol] The name of the field.
     #
@@ -34,6 +50,11 @@ module Arrow
     #     See {Arrow::DataType.resolve} how to specify data type
     #     description.
     #
+    #   @param nullable [Boolean, nil] (false) Whether the field may contain
+    #     zero or more nulls.
+    #
+    #     `nil` is processed as `true` (null may be contained).
+    #
     #   @example Create a field with {Arrow::DataType}s
     #     Arrow::Field.new("visible", Arrow::BooleanDataType.new)
     #
@@ -75,6 +96,11 @@ module Arrow
     #     See {Arrow::DataType.resolve} how to specify data type
     #     description.
     #
+    #   @option description [Boolean, nil] :nullable Whether the field
+    #     may contain zero or more nulls.
+    #
+    #     `nil` is processed as `true` (null may be contained).
+    #
     #   @example Create a field with {Arrow::DataType}s
     #     Arrow::Field.new(name: "visible",
     #                      data_type: Arrow::BooleanDataType.new)
@@ -92,6 +118,7 @@ module Arrow
         name = nil
         data_type = nil
         data_type_description = {}
+        nullable = nil
         description.each do |key, value|
           key = key.to_sym
           case key
@@ -99,20 +126,34 @@ module Arrow
             name = value
           when :data_type
             data_type = DataType.resolve(value)
+          when :nullable
+            nullable = value
           else
             data_type_description[key] = value
           end
         end
         data_type ||= DataType.resolve(data_type_description)
       when 2
+        name = args[0]
+        data_type = args[1]
+        if data_type.is_a?(Hash)
+          data_type = data_type.dup
+          nullable = data_type.delete(:nullable)
+        else
+          nullable = nil
+        end
+        data_type = DataType.resolve(data_type)
+      when 3
         name = args[0]
         data_type = DataType.resolve(args[1])
+        nullable = args[2]
       else
-        message = "wrong number of arguments (given #{n_args}, expected 1..2)"
+        message = "wrong number of arguments (given #{n_args}, expected 1..3)"
         raise ArgumentError, message
       end
+      nullable = true if nullable.nil?
 
-      initialize_raw(name, data_type)
+      initialize_raw(name, data_type, nullable)
     end
   end
 end

data/lib/arrow/list-data-type.rb

@@ -107,12 +107,7 @@ module Arrow
         description = arg
         arg = description[:field]
       end
-      if arg.is_a?(Hash)
-        field_description = arg
-        Field.new(field_description)
-      else
-        arg
-      end
+      arg
     end
   end
 end

data/lib/arrow/loader.rb

@@ -34,6 +34,7 @@ module Arrow
     end
 
     def require_libraries
+      require "arrow/array-computable"
       require "arrow/column-containable"
       require "arrow/field-containable"
       require "arrow/generic-filterable"
@@ -106,6 +107,7 @@ module Arrow
       require "arrow/source-node-options"
       require "arrow/sparse-union-data-type"
       require "arrow/string-dictionary-array-builder"
+      require "arrow/string-array-builder"
       require "arrow/struct-array"
       require "arrow/struct-array-builder"
       require "arrow/struct-data-type"
@@ -118,6 +120,7 @@ module Arrow
       require "arrow/table-saver"
       require "arrow/tensor"
       require "arrow/time"
+      require "arrow/time-unit"
       require "arrow/time32-array"
       require "arrow/time32-array-builder"
       require "arrow/time32-data-type"

data/lib/arrow/raw-table-converter.rb

@@ -35,7 +35,12 @@ module Arrow
         fields = []
         @values = []
         @raw_table.each do |name, array|
-          array = ArrayBuilder.build(array) if array.is_a?(::Array)
+          if array.respond_to?(:to_arrow_array)
+            array = array.to_arrow_array
+          else
+            array = array.to_ary if array.respond_to?(:to_ary)
+            array = ArrayBuilder.build(array)
+          end
           fields << Field.new(name.to_s, array.value_data_type)
           @values << array
         end

data/lib/arrow/raw-tensor-converter.rb

@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class RawTensorConverter
+    attr_reader :data_type
+    attr_reader :data
+    attr_reader :shape
+    attr_reader :strides
+    attr_reader :dimension_names
+    def initialize(raw_tensor,
+                   data_type: nil,
+                   shape: nil,
+                   strides: nil,
+                   dimension_names: nil)
+      @raw_tensor = raw_tensor
+      @data_type = data_type
+      @data = nil
+      @shape = shape
+      @strides = strides
+      @dimension_names = dimension_names
+      convert
+    end
+
+    private
+    def convert
+      case @raw_tensor
+      when Buffer
+        @data = @raw_tensor
+      when String
+        unless @raw_tensor.encoding == Encoding::ASCII_8BIT
+          message = "raw tensor String must be an ASCII-8BIT encoded string: " +
+                    "#{@raw_tensor.encoding.inspect}"
+          raise ArgumentError, message
+        end
+        @data = Arrow::Buffer.new(@raw_tensor)
+      else
+        @shape ||= guess_shape
+        build_buffer
+        unless @strides.nil?
+          message = "strides: is only accepted with " +
+                    "an Arrow::Buffer or String raw tensor: #{@strides.inspect}"
+          raise ArgumentError, message
+        end
+      end
+      if @shape.nil?
+        raise ArgumentError, "shape: is missing: #{@raw_tensor.inspect}"
+      end
+      if @data_type.nil?
+        raise ArgumentError, "data_type: is missing: #{@raw_tensor.inspect}"
+      end
+    end
+
+    def guess_shape
+      shape = [@raw_tensor.size]
+      target = @raw_tensor[0]
+      while target.is_a?(::Array)
+        shape << target.size
+        target = target[0]
+      end
+      shape
+    end
+
+    def build_buffer
+      if @data_type
+        @data_type = DataType.resolve(@data_type)
+        array = @data_type.build_array(@raw_tensor.flatten)
+      else
+        array = Array.new(@raw_tensor.flatten)
+        @data_type = array.value_data_type
+      end
+      @data = array.data_buffer
+    end
+  end
+end