red-arrow 6.0.1 → 9.0.0

data/test/raw-records/test-basic-arrays.rb

@@ -346,6 +346,36 @@ module RawRecordsBasicArraysTests
                    records)
     assert_equal(records, target.raw_records)
   end
+
+  def test_month_interval
+    records = [
+      [1],
+      [nil],
+      [12],
+    ]
+    target = build({column: :month_interval}, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{day: 1, millisecond: 100}],
+      [nil],
+      [{day: 2, millisecond: 300}],
+    ]
+    target = build({column: :day_time_interval}, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{month: 1, day: 1, nanosecond: 100}],
+      [nil],
+      [{month: 2, day: 3, nanosecond: 400}],
+    ]
+    target = build({column: :month_day_nano_interval}, records)
+    assert_equal(records, target.raw_records)
+  end
 end
 
 class RawRecordsRecordBatchBasicArraysTest < Test::Unit::TestCase

data/test/raw-records/test-dense-union-array.rb

@@ -359,6 +359,33 @@ module RawRecordsDenseUnionArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [{"0" => 1}],
+      [{"1" => nil}],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{"0" => {day: 1, millisecond: 100}}],
+      [{"1" => nil}],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{"0" => {month: 1, day: 1, nanosecond: 100}}],
+      [{"1" => nil}],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"0" => [true, nil, false]}],

data/test/raw-records/test-dictionary-array.rb

@@ -0,0 +1,341 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module RawRecordsDictionaryArrayTests
+  def build_record_batch(array)
+    dictionary = array.dictionary_encode
+    schema = Arrow::Schema.new(column: dictionary.value_data_type)
+    Arrow::RecordBatch.new(schema, array.length, [dictionary])
+  end
+
+  def test_null
+    records = [
+      [nil],
+      [nil],
+      [nil],
+      [nil],
+    ]
+    target = build(Arrow::NullArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_boolean
+    records = [
+      [true],
+      [nil],
+      [false],
+    ]
+    target = build(Arrow::BooleanArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_int8
+    records = [
+      [-(2 ** 7)],
+      [nil],
+      [(2 ** 7) - 1],
+    ]
+    target = build(Arrow::Int8Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_uint8
+    records = [
+      [0],
+      [nil],
+      [(2 ** 8) - 1],
+    ]
+    target = build(Arrow::UInt8Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_int16
+    records = [
+      [-(2 ** 15)],
+      [nil],
+      [(2 ** 15) - 1],
+    ]
+    target = build(Arrow::Int16Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_uint16
+    records = [
+      [0],
+      [nil],
+      [(2 ** 16) - 1],
+    ]
+    target = build(Arrow::UInt16Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_int32
+    records = [
+      [-(2 ** 31)],
+      [nil],
+      [(2 ** 31) - 1],
+    ]
+    target = build(Arrow::Int32Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_uint32
+    records = [
+      [0],
+      [nil],
+      [(2 ** 32) - 1],
+    ]
+    target = build(Arrow::UInt32Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_int64
+    records = [
+      [-(2 ** 63)],
+      [nil],
+      [(2 ** 63) - 1],
+    ]
+    target = build(Arrow::Int64Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_uint64
+    records = [
+      [0],
+      [nil],
+      [(2 ** 64) - 1],
+    ]
+    target = build(Arrow::UInt64Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_float
+    records = [
+      [-1.0],
+      [nil],
+      [1.0],
+    ]
+    target = build(Arrow::FloatArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_double
+    records = [
+      [-1.0],
+      [nil],
+      [1.0],
+    ]
+    target = build(Arrow::DoubleArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_binary
+    records = [
+      ["\x00".b],
+      [nil],
+      ["\xff".b],
+    ]
+    target = build(Arrow::BinaryArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_string
+    records = [
+      ["Ruby"],
+      [nil],
+      ["\u3042"], # U+3042 HIRAGANA LETTER A
+    ]
+    target = build(Arrow::StringArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_date32
+    records = [
+      [Date.new(1960, 1, 1)],
+      [nil],
+      [Date.new(2017, 8, 23)],
+    ]
+    target = build(Arrow::Date32Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_date64
+    records = [
+      [DateTime.new(1960, 1, 1, 2, 9, 30)],
+      [nil],
+      [DateTime.new(2017, 8, 23, 14, 57, 2)],
+    ]
+    target = build(Arrow::Date64Array.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_timestamp_second
+    records = [
+      [Time.parse("1960-01-01T02:09:30Z")],
+      [nil],
+      [Time.parse("2017-08-23T14:57:02Z")],
+    ]
+    target = build(Arrow::TimestampArray.new(:second, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_timestamp_milli
+    records = [
+      [Time.parse("1960-01-01T02:09:30.123Z")],
+      [nil],
+      [Time.parse("2017-08-23T14:57:02.987Z")],
+    ]
+    target = build(Arrow::TimestampArray.new(:milli, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_timestamp_micro
+    records = [
+      [Time.parse("1960-01-01T02:09:30.123456Z")],
+      [nil],
+      [Time.parse("2017-08-23T14:57:02.987654Z")],
+    ]
+    target = build(Arrow::TimestampArray.new(:micro, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_timestamp_nano
+    records = [
+      [Time.parse("1960-01-01T02:09:30.123456789Z")],
+      [nil],
+      [Time.parse("2017-08-23T14:57:02.987654321Z")],
+    ]
+    target = build(Arrow::TimestampArray.new(:nano, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_time32_second
+    unit = Arrow::TimeUnit::SECOND
+    records = [
+      [Arrow::Time.new(unit, 60 * 10)], # 00:10:00
+      [nil],
+      [Arrow::Time.new(unit, 60 * 60 * 2 + 9)], # 02:00:09
+    ]
+    target = build(Arrow::Time32Array.new(unit, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_time32_milli
+    unit = Arrow::TimeUnit::MILLI
+    records = [
+      [Arrow::Time.new(unit, (60 * 10) * 1000 + 123)], # 00:10:00.123
+      [nil],
+      [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987)], # 02:00:09.987
+    ]
+    target = build(Arrow::Time32Array.new(unit, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_time64_micro
+    unit = Arrow::TimeUnit::MICRO
+    records = [
+      # 00:10:00.123456
+      [Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)],
+      [nil],
+      # 02:00:09.987654
+      [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654)],
+    ]
+    target = build(Arrow::Time64Array.new(unit, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_time64_nano
+    unit = Arrow::TimeUnit::NANO
+    records = [
+      # 00:10:00.123456789
+      [Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)],
+      [nil],
+      # 02:00:09.987654321
+      [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321)],
+    ]
+    target = build(Arrow::Time64Array.new(unit, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_decimal128
+    records = [
+      [BigDecimal("92.92")],
+      [nil],
+      [BigDecimal("29.29")],
+    ]
+    data_type = Arrow::Decimal128DataType.new(8, 2)
+    target = build(Arrow::Decimal128Array.new(data_type, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_decimal256
+    records = [
+      [BigDecimal("92.92")],
+      [nil],
+      [BigDecimal("29.29")],
+    ]
+    data_type = Arrow::Decimal256DataType.new(38, 2)
+    target = build(Arrow::Decimal256Array.new(data_type, records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_interval
+    records = [
+      [1],
+      [nil],
+      [12],
+    ]
+    target = build(Arrow::MonthIntervalArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{day: 1, millisecond: 100}],
+      [nil],
+      [{day: 2, millisecond: 300}],
+    ]
+    target = build(Arrow::DayTimeIntervalArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{month: 1, day: 1, nanosecond: 100}],
+      [nil],
+      [{month: 2, day: 3, nanosecond: 400}],
+    ]
+    target = build(Arrow::MonthDayNanoIntervalArray.new(records.collect(&:first)))
+    assert_equal(records, target.raw_records)
+  end
+end
+
+class RawRecordsRecordBatchDictionaryArraysTest < Test::Unit::TestCase
+  include RawRecordsDictionaryArrayTests
+
+  def build(array)
+    build_record_batch(array)
+  end
+end
+
+class RawRecordsTableDictionaryArraysTest < Test::Unit::TestCase
+  include RawRecordsDictionaryArrayTests
+
+  def build(array)
+    build_record_batch(array).to_table
+  end
+end

data/test/raw-records/test-list-array.rb

@@ -399,6 +399,45 @@ module RawRecordsListArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [[1, nil, 12]],
+      [nil],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [
+        [
+          {day: 1, millisecond: 100},
+          nil,
+          {day: 2, millisecond: 300},
+        ]
+      ],
+      [nil],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [
+        [
+          {month: 1, day: 1, nanosecond: 100},
+          nil,
+          {month: 2, day: 3, nanosecond: 400},
+        ]
+      ],
+      [nil],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [

data/test/raw-records/test-map-array.rb

@@ -310,6 +310,43 @@ module RawRecordsMapArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [{"key1" => 1, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [
+        {
+          "key1" => {day: 1, millisecond: 100},
+          "key2" => nil,
+        },
+      ],
+      [nil],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [
+        {
+          "key1" => {month: 1, day: 1, nanosecond: 100},
+          "key2" => nil,
+        },
+      ],
+      [nil],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"key1" => [true, nil, false], "key2" => nil}],

data/test/raw-records/test-sparse-union-array.rb

@@ -349,6 +349,33 @@ module RawRecordsSparseUnionArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [{"0" => 1}],
+      [{"1" => nil}],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{"0" => {day: 1, millisecond: 100}}],
+      [{"1" => nil}],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{"0" => {month: 1, day: 1, nanosecond: 100}}],
+      [{"1" => nil}],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"0" => [true, nil, false]}],

data/test/raw-records/test-struct-array.rb

@@ -344,6 +344,36 @@ module RawRecordsStructArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [{"field" => 1}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{"field" => {day: 1, millisecond: 100}}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{"field" => {month: 1, day: 1, nanosecond: 100}}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"field" => [true, nil, false]}],

data/test/test-array-builder.rb

@@ -68,6 +68,68 @@ class ArrayBuilderTest < Test::Unit::TestCase
                      ])
       end
 
+      test("decimal + string") do
+        raw_array = [BigDecimal("10.1"), "10.1"]
+        array = Arrow::ArrayBuilder.build(raw_array)
+        assert_equal(raw_array.collect(&:to_s), array.to_a)
+      end
+
+      test("NaN") do
+        raw_array = [BigDecimal("10.1"), BigDecimal::NAN]
+        array = Arrow::ArrayBuilder.build(raw_array)
+        assert_equal(raw_array.collect(&:to_s), array.to_a)
+      end
+
+      test("Infinity") do
+        raw_array = [BigDecimal("10.1"), BigDecimal::INFINITY]
+        array = Arrow::ArrayBuilder.build(raw_array)
+        assert_equal(raw_array.collect(&:to_s), array.to_a)
+      end
+
+      test("decimal128") do
+        values = [
+          BigDecimal("10.1"),
+          BigDecimal("1.11"),
+          BigDecimal("1"),
+        ]
+        array = Arrow::Array.new(values)
+        data_type = Arrow::Decimal128DataType.new(3, 2)
+        assert_equal({
+                       data_type: data_type,
+                       values: [
+                         BigDecimal("10.1"),
+                         BigDecimal("1.11"),
+                         BigDecimal("1"),
+                       ],
+                     },
+                     {
+                       data_type: array.value_data_type,
+                       values: array.to_a,
+                     })
+      end
+
+      test("decimal256") do
+        values = [
+          BigDecimal("1" * 40 + ".1"),
+          BigDecimal("1" * 38 + ".11"),
+          BigDecimal("1" * 37),
+        ]
+        array = Arrow::Array.new(values)
+        data_type = Arrow::Decimal256DataType.new(41, 2)
+        assert_equal({
+                       data_type: data_type,
+                       values: [
+                         BigDecimal("1" * 40 + ".1"),
+                         BigDecimal("1" * 38 + ".11"),
+                         BigDecimal("1" * 37),
+                       ],
+                     },
+                     {
+                       data_type: array.value_data_type,
+                       values: array.to_a,
+                     })
+      end
+
       test("list<boolean>s") do
         assert_build(Arrow::ArrayBuilder,
                      [

data/test/test-chunked-array.rb

@@ -180,4 +180,10 @@ class ChunkedArrayTest < Test::Unit::TestCase
                    @chunked_array.take(indices))
     end
   end
+
+  test("#cast") do
+    chunked_array = Arrow::ChunkedArray.new([[1, nil, 3]])
+    assert_equal(Arrow::ChunkedArray.new([["1", nil, "3"]]),
+                 chunked_array.cast(:string))
+  end
 end

data/test/test-column.rb

@@ -89,4 +89,35 @@ class ColumnTest < Test::Unit::TestCase
       end
     end
   end
+
+  test("#count") do
+    table = Arrow::Table.new("revenue" => [1, nil, 3])
+    assert_equal(2, table["revenue"].count)
+  end
+
+  test("#min") do
+    table = Arrow::Table.new("revenue" => [1, 2, 3])
+    assert_equal(1, table["revenue"].min)
+  end
+
+  test("#max") do
+    table = Arrow::Table.new("revenue" => [1, 2, 3])
+    assert_equal(3, table["revenue"].max)
+  end
+
+  test("#sum") do
+    table = Arrow::Table.new("revenue" => [1, 2, 3])
+    assert_equal(6, table["revenue"].sum)
+  end
+
+  test("#uniq") do
+    table = Arrow::Table.new("revenue" => [1, 2, 2])
+    assert_equal([1, 2], table["revenue"].uniq)
+  end
+
+  test("#cast") do
+    table = Arrow::Table.new("revenue" => [1, nil, 3])
+    assert_equal(Arrow::ChunkedArray.new([["1", nil, "3"]]),
+                 table["revenue"].cast(:string))
+  end
 end

data/test/test-decimal128-array-builder.rb

@@ -55,6 +55,20 @@ class Decimal128ArrayBuilderTest < Test::Unit::TestCase
       assert_equal(BigDecimal("10.1"),
                    array[0])
     end
+
+    test("BigDecimal::NAN") do
+      message = "can't use NaN as an Arrow::Decimal128Array value"
+      assert_raise(FloatDomainError.new(message)) do
+        @builder.append_value(BigDecimal::NAN)
+      end
+    end
+
+    test("BigDecimal::INFINITY") do
+      message = "can't use Infinity as an Arrow::Decimal128Array value"
+      assert_raise(FloatDomainError.new(message)) do
+        @builder.append_value(BigDecimal::INFINITY)
+      end
+    end
   end
 
   sub_test_case("#append_values") do