red-arrow 3.0.0 → 6.0.0

data/test/test-array.rb

@@ -64,6 +64,40 @@ class ArrayTest < Test::Unit::TestCase
       end
     end
 
+    sub_test_case("#equal_array?") do
+      test("no options") do
+        array1 = Arrow::FloatArray.new([1.1, Float::NAN])
+        array2 = Arrow::FloatArray.new([1.1, Float::NAN])
+        assert do
+          not array1.equal_array?(array2)
+        end
+      end
+
+      test("approx") do
+        array1 = Arrow::FloatArray.new([1.1])
+        array2 = Arrow::FloatArray.new([1.100001])
+        assert do
+          array1.equal_array?(array2, approx: true)
+        end
+      end
+
+      test("nans-equal") do
+        array1 = Arrow::FloatArray.new([1.1, Float::NAN])
+        array2 = Arrow::FloatArray.new([1.1, Float::NAN])
+        assert do
+          array1.equal_array?(array2, nans_equal: true)
+        end
+      end
+
+      test("absolute-tolerance") do
+        array1 = Arrow::FloatArray.new([1.1])
+        array2 = Arrow::FloatArray.new([1.101])
+        assert do
+          array1.equal_array?(array2, approx: true, absolute_tolerance: 0.01)
+        end
+      end
+    end
+
     sub_test_case("#cast") do
       test("Symbol") do
         assert_equal(Arrow::Int32Array.new([1, 2, 3]),
@@ -168,4 +202,124 @@ class ArrayTest < Test::Unit::TestCase
                    @array.is_in(right))
     end
   end
+
+  sub_test_case("#concatenate") do
+    test("Arrow::Array: same") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,5, 6]),
+                   Arrow::Int32Array.new([1, 2, nil]).
+                     concatenate(Arrow::Int32Array.new([4, 5]),
+                                 Arrow::Int32Array.new([6])))
+    end
+
+    test("Arrow::Array: castable") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,5, 6]),
+                   Arrow::Int32Array.new([1, 2, nil]).
+                     concatenate(Arrow::Int8Array.new([4, 5]),
+                                 Arrow::UInt32Array.new([6])))
+    end
+
+    test("Arrow::Array: non-castable") do
+      assert_raise(Arrow::Error::Invalid) do
+        Arrow::Int32Array.new([1, 2, nil]).
+          concatenate(Arrow::StringArray.new(["X"]))
+      end
+    end
+
+    test("Array") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,nil, 6]),
+                   Arrow::Int32Array.new([1, 2, nil]).
+                     concatenate([4, nil],
+                                 [6]))
+    end
+
+    test("invalid") do
+      message = "[array][resolve] can't build int32 array: 4"
+      assert_raise(ArgumentError.new(message)) do
+        Arrow::Int32Array.new([1, 2, nil]).
+          concatenate(4)
+      end
+    end
+  end
+
+  sub_test_case("#+") do
+    test("Arrow::Array: same") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,5, 6]),
+                   Arrow::Int32Array.new([1, 2, nil]) +
+                   Arrow::Int32Array.new([4, 5, 6]))
+    end
+
+    test("Arrow::Array: castable") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,5, 6]),
+                   Arrow::Int32Array.new([1, 2, nil]) +
+                   Arrow::Int8Array.new([4, 5, 6]))
+    end
+
+    test("Arrow::Array: non-castable") do
+      assert_raise(Arrow::Error::Invalid) do
+        Arrow::Int32Array.new([1, 2, nil]) +
+          Arrow::StringArray.new(["X"])
+      end
+    end
+
+    test("Array") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil, 4 ,nil, 6]),
+                   Arrow::Int32Array.new([1, 2, nil]) +
+                   [4, nil, 6])
+    end
+
+    test("invalid") do
+      message = "[array][resolve] can't build int32 array: 4"
+      assert_raise(ArgumentError.new(message)) do
+        Arrow::Int32Array.new([1, 2, nil]) + 4
+      end
+    end
+  end
+
+  sub_test_case("#resolve") do
+    test("Arrow::Array: same") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil]),
+                   Arrow::Int32Array.new([]).
+                     resolve(Arrow::Int32Array.new([1, 2, nil])))
+    end
+
+    test("Arrow::Array: castable") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil]),
+                   Arrow::Int32Array.new([]).
+                     resolve(Arrow::Int8Array.new([1, 2, nil])))
+    end
+
+    test("Arrow::Array: non-castable") do
+      assert_raise(Arrow::Error::Invalid) do
+        Arrow::Int32Array.new([]) +
+          Arrow::StringArray.new(["X"])
+      end
+    end
+
+    test("Array: non-parametric") do
+      assert_equal(Arrow::Int32Array.new([1, 2, nil]),
+                   Arrow::Int32Array.new([]).
+                     resolve([1, 2, nil]))
+    end
+
+    test("Array: parametric") do
+      list_data_type = Arrow::ListDataType.new(name: "visible", type: :boolean)
+      list_array = Arrow::ListArray.new(list_data_type, [])
+      assert_equal(Arrow::ListArray.new(list_data_type,
+                                        [
+                                          [true, false],
+                                          nil,
+                                        ]),
+                   list_array.resolve([
+                                        [true, false],
+                                        nil,
+                                      ]))
+    end
+
+    test("invalid") do
+      message = "[array][resolve] can't build int32 array: 4"
+      assert_raise(ArgumentError.new(message)) do
+        Arrow::Int32Array.new([]).resolve(4)
+      end
+    end
+  end
 end

data/test/test-binary-dictionary-array-builder.rb

@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class BinaryDictionaryArrayBuilderTest < Test::Unit::TestCase
+  def setup
+    @builder = Arrow::BinaryDictionaryArrayBuilder.new
+  end
+
+  sub_test_case("#append_values") do
+    test("[nil]") do
+      @builder.append_values([nil])
+      array = @builder.finish
+      assert_equal([
+                     [],
+                     [nil],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+
+    test("[String]") do
+      @builder.append_values(["he\xffllo"])
+      array = @builder.finish
+      assert_equal([
+                     ["he\xffllo".b],
+                     [0],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+
+    test("[Symbol]") do
+      @builder.append_values([:hello])
+      array = @builder.finish
+      assert_equal([
+                     ["hello"],
+                     [0],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+
+    test("[nil, String, Symbol]") do
+      @builder.append_values([
+                               nil,
+                               "He\xffllo",
+                               :world,
+                               "world",
+                             ])
+      array = @builder.finish
+      assert_equal([
+                     ["He\xffllo".b, "world"],
+                     [nil, 0, 1, 1],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+
+    test("is_valids") do
+      @builder.append_values([
+                               "He\xffllo",
+                               :world,
+                               :goodbye,
+                             ],
+                             [
+                               true,
+                               false,
+                               true,
+                             ])
+      array = @builder.finish
+      assert_equal([
+                     ["He\xffllo".b, "goodbye"],
+                     [0, nil, 1],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+  end
+end

data/test/test-boolean-scalar.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class BooleanScalarTest < Test::Unit::TestCase
+  def setup
+    @scalar = Arrow::BooleanScalar.new(true)
+  end
+
+  test("#value") do
+    assert_equal(true, @scalar.value)
+  end
+end

data/test/test-csv-loader.rb

@@ -86,20 +86,20 @@ class CSVLoaderTest < Test::Unit::TestCase
     test("null: with double quote") do
       path = fixture_path("null-with-double-quote.csv").to_s
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	name	score
-0	alice	   10
-1	bob 	     
-2	chris	   -1
+	name	 score
+0	alice	    10
+1	bob 	(null)
+2	chris	    -1
       TABLE
     end
 
     test("null: without double quote") do
       path = fixture_path("null-without-double-quote.csv").to_s
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	name	score
-0	alice	   10
-1	bob 	     
-2	chris	   -1
+	name	 score
+0	alice	    10
+1	bob 	(null)
+2	chris	    -1
       TABLE
     end
 

data/test/test-decimal128-data-type.rb

@@ -18,12 +18,12 @@
 class Decimal128DataTypeTest < Test::Unit::TestCase
   sub_test_case(".new") do
     test("ordered arguments") do
-      assert_equal("decimal(8, 2)",
+      assert_equal("decimal128(8, 2)",
                    Arrow::Decimal128DataType.new(8, 2).to_s)
     end
 
     test("description") do
-      assert_equal("decimal(8, 2)",
+      assert_equal("decimal128(8, 2)",
                    Arrow::Decimal128DataType.new(precision: 8,
                                                  scale: 2).to_s)
     end

data/test/test-expression.rb

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestExpression < Test::Unit::TestCase
+  sub_test_case(".try_convert") do
+    test("Symbol") do
+      assert_equal(Arrow::FieldExpression.new("visible"),
+                   Arrow::Expression.try_convert(:visible))
+    end
+
+    test("[String]") do
+      assert_equal(Arrow::CallExpression.new("func", []),
+                   Arrow::Expression.try_convert(["func"]))
+    end
+
+    test("[Symbol]") do
+      assert_equal(Arrow::CallExpression.new("func", []),
+                   Arrow::Expression.try_convert([:func]))
+    end
+
+    test("[String, String]") do
+      assert_equal(Arrow::CallExpression.new("func", ["argument1"]),
+                   Arrow::Expression.try_convert(["func", "argument1"]))
+    end
+  end
+end

data/test/test-float-scalar.rb

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class FloatScalarTest < Test::Unit::TestCase
+  sub_test_case("#equal_scalar?") do
+    test("no options") do
+      scalar1 = Arrow::FloatScalar.new(1.1)
+      scalar2 = Arrow::FloatScalar.new(1.1000001)
+      assert do
+        not scalar1.equal_scalar?(scalar2)
+      end
+    end
+
+    test(":approx") do
+      scalar1 = Arrow::FloatScalar.new(1.1)
+      scalar2 = Arrow::FloatScalar.new(1.1000001)
+      assert do
+        scalar1.equal_scalar?(scalar2, approx: true)
+      end
+    end
+
+    test(":absolute_tolerance") do
+      scalar1 = Arrow::FloatScalar.new(1.1)
+      scalar2 = Arrow::FloatScalar.new(1.1001)
+      assert do
+        scalar1.equal_scalar?(scalar2,
+                              approx: true,
+                              absolute_tolerance: 0.001)
+      end
+    end
+  end
+end

data/test/test-function.rb

@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class FunctionTest < Test::Unit::TestCase
+  sub_test_case("#execute") do
+    test("Arrow::Array") do
+      or_function = Arrow::Function.find("or")
+      args = [
+        Arrow::BooleanArray.new([true, false, false]),
+        Arrow::BooleanArray.new([true, false, true]),
+      ]
+      assert_equal([true, false, true],
+                   or_function.execute(args).value.to_a)
+    end
+
+    test("Array") do
+      or_function = Arrow::Function.find("or")
+      args = [
+        [true, false, false],
+        [true, false, true],
+      ]
+      assert_equal([true, false, true],
+                   or_function.execute(args).value.to_a)
+    end
+
+    test("Arrow::ChunkedArray") do
+      or_function = Arrow::Function.find("or")
+      args = [
+        Arrow::ChunkedArray.new([
+                                  Arrow::BooleanArray.new([true]),
+                                  Arrow::BooleanArray.new([false, false]),
+                                ]),
+        Arrow::ChunkedArray.new([
+                                  Arrow::BooleanArray.new([true, false]),
+                                  Arrow::BooleanArray.new([true]),
+                                ]),
+      ]
+      assert_equal([true, false, true],
+                   or_function.execute(args).value.to_a)
+    end
+
+    test("Arrow::Scalar") do
+      add_function = Arrow::Function.find("add")
+      args = [
+        Arrow::Int8Array.new([1, 2, 3]),
+        Arrow::Int8Scalar.new(5),
+      ]
+      assert_equal([6, 7, 8],
+                   add_function.execute(args).value.to_a)
+    end
+
+    test("Integer") do
+      add_function = Arrow::Function.find("add")
+      args = [
+        [1, 2, 3],
+        5,
+      ]
+      assert_equal([6, 7, 8],
+                   add_function.execute(args).value.to_a)
+    end
+
+    test("Float") do
+      add_function = Arrow::Function.find("add")
+      args = [
+        [1, 2, 3],
+        5.1,
+      ]
+      assert_equal([6.1, 7.1, 8.1],
+                   add_function.execute(args).value.to_a)
+    end
+
+    test("true") do
+      and_function = Arrow::Function.find("and")
+      args = [
+        Arrow::BooleanArray.new([true, false, false]),
+        true,
+      ]
+      assert_equal([true, false, false],
+                   and_function.execute(args).value.to_a)
+    end
+
+    test("false") do
+      or_function = Arrow::Function.find("or")
+      args = [
+        Arrow::BooleanArray.new([true, false, false]),
+        false,
+      ]
+      assert_equal([true, false, false],
+                   or_function.execute(args).value.to_a)
+    end
+
+    test("String") do
+      ascii_upper_function = Arrow::Function.find("ascii_upper")
+      args = [
+        "Hello",
+      ]
+      assert_equal("HELLO",
+                   ascii_upper_function.execute(args).value.to_s)
+    end
+
+    test("Date") do
+      cast_function = Arrow::Function.find("cast")
+      date = Date.new(2021, 6, 12)
+      args = [date]
+      options = Arrow::CastOptions.new
+      options.to_data_type = Arrow::TimestampDataType.new(:second)
+      time = Time.utc(date.year,
+                      date.month,
+                      date.day)
+      assert_equal(Arrow::TimestampScalar.new(options.to_data_type,
+                                              time.to_i),
+                   cast_function.execute(args, options).value)
+    end
+
+    test("Arrow::Time: second") do
+      cast_function = Arrow::Function.find("cast")
+      arrow_time = Arrow::Time.new(Arrow::TimeUnit::SECOND,
+                                   # 00:10:00
+                                   60 * 10)
+      args = [arrow_time]
+      options = Arrow::CastOptions.new
+      options.to_data_type = Arrow::Time64DataType.new(:micro)
+      assert_equal(Arrow::Time64Scalar.new(options.to_data_type,
+                                           # 00:10:00.000000
+                                           60 * 10 * 1000 * 1000),
+                   cast_function.execute(args, options).value)
+    end
+
+    test("Arrow::Time: micro") do
+      cast_function = Arrow::Function.find("cast")
+      arrow_time = Arrow::Time.new(Arrow::TimeUnit::MICRO,
+                                   # 00:10:00.000000
+                                   60 * 10 * 1000 * 1000)
+      args = [arrow_time]
+      options = Arrow::CastOptions.new
+      options.to_data_type = Arrow::Time32DataType.new(:second)
+      options.allow_time_truncate = true
+      assert_equal(Arrow::Time32Scalar.new(options.to_data_type,
+                                           # 00:10:00
+                                           60 * 10),
+                   cast_function.execute(args, options).value)
+    end
+
+    test("Time") do
+      cast_function = Arrow::Function.find("cast")
+      time = Time.utc(2021, 6, 12, 1, 2, 3, 1)
+      args = [time]
+      options = Arrow::CastOptions.new
+      options.to_data_type = Arrow::TimestampDataType.new(:second)
+      options.allow_time_truncate = true
+      time = Time.utc(time.year,
+                      time.month,
+                      time.day,
+                      time.hour,
+                      time.min,
+                      time.sec)
+      assert_equal(Arrow::TimestampScalar.new(options.to_data_type,
+                                              time.to_i),
+                   cast_function.execute(args, options).value)
+    end
+  end
+end