red-arrow 8.0.0 → 24.0.0

data/lib/arrow/column.rb

@@ -27,6 +27,23 @@ module Arrow
       @index = index
       @field = @container.schema[@index]
       @data = @container.get_column_data(@index)
+      @container.share_input(@data)
+    end
+
+    def to_arrow
+      @data
+    end
+
+    def to_arrow_array
+      @data.to_arrow_array
+    end
+
+    def to_arrow_chunked_array
+      @data.to_arrow_chunked_array
+    end
+
+    def values
+      @data.values
     end
 
     def name
@@ -57,11 +74,11 @@ module Arrow
       @data.reverse_each(&block)
     end
 
-    def n_rows
-      @data.n_rows
+    def size
+      @data.size
     end
-    alias_method :size, :n_rows
-    alias_method :length, :n_rows
+    alias_method :length, :size
+    alias_method :n_rows, :size
 
     def n_nulls
       @data.n_nulls
@@ -72,5 +89,33 @@ module Arrow
         @field == other.field and
         @data == other.data
     end
+
+    def count(options: nil)
+      @data.count(options: options)
+    end
+
+    def sum(options: nil)
+      @data.sum(options: options)
+    end
+
+    def min(options: nil)
+      @data.min(options: options)
+    end
+
+    def max(options: nil)
+      @data.max(options: options)
+    end
+
+    def unique
+      @data.unique
+    end
+
+    def uniq
+      @data.uniq
+    end
+
+    def cast(target_data_type, options: nil)
+      @data.cast(target_data_type, options: options)
+    end
   end
 end

data/{test/test-tensor.rb → lib/arrow/csv-write-options.rb}

@@ -15,42 +15,39 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class TensorTest < Test::Unit::TestCase
-  sub_test_case("instance methods") do
-    def setup
-      raw_data = [
-        1, 2,
-        3, 4,
-
-        5, 6,
-        7, 8,
-
-        9, 10,
-        11, 12,
-      ]
-      data = Arrow::Buffer.new(raw_data.pack("c*"))
-      shape = [3, 2, 2]
-      strides = []
-      names = ["a", "b", "c"]
-      @tensor = Arrow::Tensor.new(Arrow::Int8DataType.new,
-                                  data,
-                                  shape,
-                                  strides,
-                                  names)
-    end
-
-    sub_test_case("#==") do
-      test("Arrow::Tensor") do
-        assert do
-          @tensor == @tensor
+module Arrow
+  class CSVWriteOptions
+    class << self
+      def try_convert(value)
+        case value
+        when Hash
+          options = new
+          value.each do |k, v|
+            options.public_send("#{k}=", v)
+          end
+          options
+        else
+          nil
         end
       end
+    end
+    
+    alias_method :delimiter_raw, :delimiter
+    def delimiter
+      delimiter_raw.chr
+    end
 
-      test("not Arrow::Tensor") do
-        assert do
-          not (@tensor == 29)
+    alias_method :delimiter_raw=, :delimiter=
+    def delimiter=(delimiter)
+      case delimiter
+      when String
+        if delimiter.bytesize != 1
+          message = "delimiter must be 1 byte character: #{delimiter.inspect}"
+          raise ArgumentError, message
         end
+        delimiter = delimiter.ord
       end
+      self.delimiter_raw = delimiter
     end
   end
 end

data/lib/arrow/data-type.rb

@@ -110,7 +110,7 @@ module Arrow
               description[key] = value
             end
           end
-          if type.nil?
+          if type.nil? and self == DataType
             message =
               "data type description must have :type value: #{data_type.inspect}"
             raise ArgumentError, message
@@ -152,6 +152,7 @@ module Arrow
 
       private
       def resolve_class(data_type)
+        return self if data_type.nil?
         components = data_type.to_s.split("_").collect(&:capitalize)
         data_type_name = components.join.gsub(/\AUint/, "UInt")
         data_type_class_name = "#{data_type_name}DataType"
@@ -187,12 +188,25 @@ module Arrow
       end
     end
 
-    def build_array(values)
+    def array_class
       base_name = self.class.name.gsub(/DataType\z/, "")
-      builder_class = self.class.const_get("#{base_name}ArrayBuilder")
+      ::Arrow.const_get("#{base_name}Array")
+    end
+
+    def build_array(values)
+      builder_class = array_class.builder_class
       args = [values]
       args.unshift(self) unless builder_class.buildable?(args)
       builder_class.build(*args)
     end
+
+    # @return [Arrow::Scalar} A corresponding {Arrow::Scalar} class
+    #   for this data type.
+    #
+    # @since 12.0.0
+    def scalar_class
+      base_name = self.class.name.gsub(/DataType\z/, "")
+      ::Arrow.const_get("#{base_name}Scalar")
+    end
   end
 end

data/lib/arrow/decimal128-array-builder.rb

@@ -42,16 +42,26 @@ module Arrow
     end
 
     private
+    def precision
+      @precision ||= value_data_type.precision
+    end
+
+    def scale
+      @scale ||= value_data_type.scale
+    end
+
     def normalize_value(value)
       case value
-      when String
-        Decimal128.new(value)
-      when Float
-        Decimal128.new(value.to_s)
       when BigDecimal
-        Decimal128.new(value.to_s)
+        if value.nan? or value.infinite?
+          message = "can't use #{value} as an Arrow::Decimal128Array value"
+          raise FloatDomainError, message
+        end
+        integer, decimal = value.to_s("f").split(".", 2)
+        decimal = decimal[0, scale].ljust(scale, "0")
+        Decimal128.new("#{integer}.#{decimal}")
       else
-        value
+        Decimal128.try_convert(value) || value
       end
     end
   end

data/lib/arrow/decimal128.rb

@@ -17,6 +17,20 @@
 
 module Arrow
   class Decimal128
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        when Float
+          new(value.to_s)
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :to_s_raw, :to_s
 
     # @overload to_s

data/lib/arrow/decimal256-array-builder.rb

@@ -45,16 +45,26 @@ module Arrow
     end
 
     private
+    def precision
+      @precision ||= value_data_type.precision
+    end
+
+    def scale
+      @scale ||= value_data_type.scale
+    end
+
     def normalize_value(value)
       case value
-      when String
-        Decimal256.new(value)
-      when Float
-        Decimal256.new(value.to_s)
       when BigDecimal
-        Decimal256.new(value.to_s)
+        if value.nan? or value.infinite?
+          message = "can't use #{value} as an Arrow::Decimal256Array value"
+          raise FloatDomainError, message
+        end
+        integer, decimal = value.to_s("f").split(".", 2)
+        decimal = decimal[0, scale].ljust(scale, "0")
+        Decimal256.new("#{integer}.#{decimal}")
       else
-        value
+        Decimal256.try_convert(value) || value
       end
     end
   end

data/lib/arrow/decimal256.rb

@@ -17,6 +17,20 @@
 
 module Arrow
   class Decimal256
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        when Float
+          new(value.to_s)
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :to_s_raw, :to_s
 
     # @overload to_s

data/{test/test-float-scalar.rb → lib/arrow/dense-union-array-builder.rb}

@@ -15,31 +15,34 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class FloatScalarTest < Test::Unit::TestCase
-  sub_test_case("#equal_scalar?") do
-    test("no options") do
-      scalar1 = Arrow::FloatScalar.new(1.1)
-      scalar2 = Arrow::FloatScalar.new(1.1000001)
-      assert do
-        not scalar1.equal_scalar?(scalar2)
-      end
-    end
-
-    test(":approx") do
-      scalar1 = Arrow::FloatScalar.new(1.1)
-      scalar2 = Arrow::FloatScalar.new(1.1000001)
-      assert do
-        scalar1.equal_scalar?(scalar2, approx: true)
-      end
-    end
+module Arrow
+  class DenseUnionArrayBuilder
+    alias_method :append_value_raw, :append_value
 
-    test(":absolute_tolerance") do
-      scalar1 = Arrow::FloatScalar.new(1.1)
-      scalar2 = Arrow::FloatScalar.new(1.1001)
-      assert do
-        scalar1.equal_scalar?(scalar2,
-                              approx: true,
-                              absolute_tolerance: 0.001)
+    # @overload append_value
+    #
+    #   Starts appending an union record. You need to append values of
+    #   fields.
+    #
+    # @overload append_value(value)
+    #
+    #   Appends an union record including values of fields.
+    #
+    #   @param value [nil, Hash] The union record value.
+    #
+    #     If this is `nil`, the union record is null.
+    #
+    #     If this is `Hash`, it's  values of fields.
+    #
+    # @since 12.0.0
+    def append_value(value)
+      if value.nil?
+        append_null
+      else
+        key = value.keys[0]
+        child_info = child_infos[key]
+        append_value_raw(child_info[:id])
+        child_info[:builder].append(value.values[0])
       end
     end
   end

data/{test/test-boolean-scalar.rb → lib/arrow/dense-union-array.rb}

@@ -15,12 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class BooleanScalarTest < Test::Unit::TestCase
-  def setup
-    @scalar = Arrow::BooleanScalar.new(true)
-  end
-
-  test("#value") do
-    assert_equal(true, @scalar.value)
+module Arrow
+  class DenseUnionArray
+    def get_value(i)
+      child_id = get_child_id(i)
+      field = fields[child_id]
+      field[get_value_offset(i)]
+    end
   end
 end

data/lib/arrow/duration-array-builder.rb

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DurationArrayBuilder
+    class << self
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+  end
+end

data/lib/arrow/duration-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DurationArray
+    def unit
+      @unit ||= value_data_type.unit
+    end
+  end
+end

data/lib/arrow/duration-data-type.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DurationDataType
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Symbol, Arrow::TimeUnit
+          new(value)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/arrow/expression.rb

@@ -31,10 +31,14 @@ module Arrow
           else
             return nil
           end
+          options = nil
           if arguments.last.is_a?(FunctionOptions)
             options = arguments.pop
-          else
-            options = nil
+          elsif arguments.last.is_a?(Hash)
+            function = Function.find(function_name)
+            if function
+              options = function.resolve_options(arguments.pop)
+            end
           end
           CallExpression.new(function_name, arguments, options)
         else

data/lib/arrow/field-containable.rb

@@ -29,7 +29,7 @@ module Arrow
         return nil if index < 0 or index >= n_fields
         get_field(index)
       else
-        message = "field name or index must be String, Symbol or Integer"
+        message = +"field name or index must be String, Symbol or Integer"
         message << ": <#{name_or_index.inspect}>"
         raise ArgumentError, message
       end

data/lib/arrow/field.rb

@@ -17,12 +17,28 @@
 
 module Arrow
   class Field
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          begin
+            new(value)
+          rescue ArgumentError
+            nil
+          end
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :initialize_raw, :initialize
     private :initialize_raw
 
     # Creates a new {Arrow::Field}.
     #
-    # @overload initialize(name, data_type)
+    # @overload initialize(name, data_type, nullable=false)
     #
     #   @param name [String, Symbol] The name of the field.
     #
@@ -34,6 +50,11 @@ module Arrow
     #     See {Arrow::DataType.resolve} how to specify data type
     #     description.
     #
+    #   @param nullable [Boolean, nil] (false) Whether the field may contain
+    #     zero or more nulls.
+    #
+    #     `nil` is processed as `true` (null may be contained).
+    #
     #   @example Create a field with {Arrow::DataType}s
     #     Arrow::Field.new("visible", Arrow::BooleanDataType.new)
     #
@@ -75,6 +96,11 @@ module Arrow
     #     See {Arrow::DataType.resolve} how to specify data type
     #     description.
     #
+    #   @option description [Boolean, nil] :nullable Whether the field
+    #     may contain zero or more nulls.
+    #
+    #     `nil` is processed as `true` (null may be contained).
+    #
     #   @example Create a field with {Arrow::DataType}s
     #     Arrow::Field.new(name: "visible",
     #                      data_type: Arrow::BooleanDataType.new)
@@ -92,6 +118,7 @@ module Arrow
         name = nil
         data_type = nil
         data_type_description = {}
+        nullable = nil
         description.each do |key, value|
           key = key.to_sym
           case key
@@ -99,20 +126,34 @@ module Arrow
             name = value
           when :data_type
             data_type = DataType.resolve(value)
+          when :nullable
+            nullable = value
           else
             data_type_description[key] = value
           end
         end
         data_type ||= DataType.resolve(data_type_description)
       when 2
+        name = args[0]
+        data_type = args[1]
+        if data_type.is_a?(Hash)
+          data_type = data_type.dup
+          nullable = data_type.delete(:nullable)
+        else
+          nullable = nil
+        end
+        data_type = DataType.resolve(data_type)
+      when 3
         name = args[0]
         data_type = DataType.resolve(args[1])
+        nullable = args[2]
       else
-        message = "wrong number of arguments (given #{n_args}, expected 1..2)"
+        message = "wrong number of arguments (given #{n_args}, expected 1..3)"
         raise ArgumentError, message
       end
+      nullable = true if nullable.nil?
 
-      initialize_raw(name, data_type)
+      initialize_raw(name, data_type, nullable)
     end
   end
 end

data/lib/arrow/fixed-size-list-array-builder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class FixedSizeListArrayBuilder
+    class << self
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+
+    prepend ListValuesAppendable
+  end
+end