red-arrow 10.0.0 → 16.0.0

data/test/raw-records/test-sparse-union-array.rb

@@ -66,12 +66,25 @@ module RawRecordsSparseUnionArrayTests
                            [union_array])
   end
 
+  def remove_field_names(records)
+    records.collect do |record|
+      record.collect do |column|
+        if column.nil?
+          column
+        else
+          column.values[0]
+        end
+      end
+    end
+  end
+
   def test_null
     records = [
       [{"0" => nil}],
     ]
     target = build(:null, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_boolean
@@ -80,7 +93,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:boolean, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_int8
@@ -89,7 +103,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:int8, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_uint8
@@ -98,7 +113,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:uint8, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_int16
@@ -107,7 +123,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:int16, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_uint16
@@ -116,7 +133,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:uint16, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_int32
@@ -125,7 +143,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:int32, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_uint32
@@ -134,7 +153,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:uint32, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_int64
@@ -143,7 +163,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:int64, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_uint64
@@ -152,7 +173,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:uint64, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_float
@@ -161,7 +183,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:float, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_double
@@ -170,7 +193,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:double, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_binary
@@ -179,7 +203,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:binary, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_string
@@ -188,7 +213,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:string, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_date32
@@ -197,7 +223,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:date32, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_date64
@@ -206,7 +233,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:date64, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_timestamp_second
@@ -219,7 +247,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :second,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_timestamp_milli
@@ -232,7 +261,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :milli,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_timestamp_micro
@@ -245,7 +275,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :micro,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_timestamp_nano
@@ -258,7 +289,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :nano,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_time32_second
@@ -273,7 +305,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :second,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_time32_milli
@@ -288,7 +321,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :milli,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_time64_micro
@@ -303,7 +337,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :micro,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_time64_nano
@@ -318,7 +353,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :nano,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_decimal128
@@ -332,7 +368,8 @@ module RawRecordsSparseUnionArrayTests
                      scale: 2,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_decimal256
@@ -346,7 +383,8 @@ module RawRecordsSparseUnionArrayTests
                      scale: 2,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_month_interval
@@ -355,7 +393,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:month_interval, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_day_time_interval
@@ -364,7 +403,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:day_time_interval, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_month_day_nano_interval
@@ -373,7 +413,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:month_day_nano_interval, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_list
@@ -389,7 +430,8 @@ module RawRecordsSparseUnionArrayTests
                      },
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_struct
@@ -408,7 +450,8 @@ module RawRecordsSparseUnionArrayTests
                      ],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_map
@@ -422,14 +465,15 @@ module RawRecordsSparseUnionArrayTests
                      item: :boolean,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_sparse_union
-    omit("Need to add support for SparseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
       [{"1" => nil}],
+      [{"0" => {"field2" => 29}}],
       [{"0" => {"field2" => nil}}],
     ]
     target = build({
@@ -447,14 +491,15 @@ module RawRecordsSparseUnionArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(remove_field_names(records)),
+                 target.raw_records)
   end
 
   def test_dense_union
-    omit("Need to add support for DenseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
       [{"1" => nil}],
+      [{"0" => {"field2" => 29}}],
       [{"0" => {"field2" => nil}}],
     ]
     target = build({
@@ -472,25 +517,25 @@ module RawRecordsSparseUnionArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(remove_field_names(records)),
+                 target.raw_records)
   end
 
   def test_dictionary
-    omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [{"0" => "Ruby"}],
       [{"1" => nil}],
       [{"0" => "GLib"}],
     ]
-    dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
     target = build({
                      type: :dictionary,
                      index_data_type: :int8,
-                     dictionary: dictionary,
-                     ordered: true,
+                     value_data_type: :string,
+                     ordered: false,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 end
 

data/test/raw-records/test-struct-array.rb

@@ -426,12 +426,26 @@ module RawRecordsStructArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def remove_union_field_names(records)
+    records.collect do |record|
+      record.collect do |column|
+        if column.nil?
+          column
+        else
+          value = column["field"]
+          value = value.values[0] unless value.nil?
+          {"field" => value}
+        end
+      end
+    end
+  end
+
   def test_sparse_union
-    omit("Need to add support for SparseUnionArrayBuilder")
     records = [
       [{"field" => {"field1" => true}}],
       [nil],
       [{"field" => nil}],
+      [{"field" => {"field2" => 29}}],
       [{"field" => {"field2" => nil}}],
     ]
     target = build({
@@ -449,15 +463,16 @@ module RawRecordsStructArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_union_field_names(records),
+                 target.raw_records)
   end
 
   def test_dense_union
-    omit("Need to add support for DenseUnionArrayBuilder")
     records = [
       [{"field" => {"field1" => true}}],
       [nil],
       [{"field" => nil}],
+      [{"field" => {"field2" => 29}}],
       [{"field" => {"field2" => nil}}],
     ]
     target = build({
@@ -475,23 +490,22 @@ module RawRecordsStructArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_union_field_names(records),
+                 target.raw_records)
   end
 
   def test_dictionary
-    omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [{"field" => "Ruby"}],
       [nil],
       [{"field" => nil}],
       [{"field" => "GLib"}],
     ]
-    dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
     target = build({
                      type: :dictionary,
                      index_data_type: :int8,
-                     dictionary: dictionary,
-                     ordered: true,
+                     value_data_type: :string,
+                     ordered: false,
                    },
                    records)
     assert_equal(records, target.raw_records)

data/test/test-array.rb

@@ -322,4 +322,11 @@ class ArrayTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case("#index") do
+    test("Integer") do
+      assert_equal(2,
+                   Arrow::Int32Array.new([1, 2, 3, 4, 5]).index(3))
+    end
+  end
 end

data/test/test-chunked-array.rb

@@ -186,4 +186,13 @@ class ChunkedArrayTest < Test::Unit::TestCase
     assert_equal(Arrow::ChunkedArray.new([["1", nil, "3"]]),
                  chunked_array.cast(:string))
   end
+
+  test("#index") do
+    arrays = [
+      Arrow::Int32Array.new([1, 2]),
+      Arrow::Int32Array.new([3, 4, 5]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(arrays)
+    assert_equal(2, chunked_array.index(3))
+  end
 end

data/test/test-csv-loader.rb

@@ -17,6 +17,7 @@
 
 class CSVLoaderTest < Test::Unit::TestCase
   include Helper::Fixture
+  include Helper::Omittable
 
   def load_csv(input)
     Arrow::CSVLoader.load(input, skip_lines: /^#/)
@@ -246,5 +247,43 @@ count
                             encoding: encoding,
                             compression: :gzip))
     end
+
+    sub_test_case(":timestamp_parsers") do
+      test(":iso8601") do
+        require_glib(2, 58, 0)
+        data_type = Arrow::TimestampDataType.new(:second,
+                                                 GLib::TimeZone.new("UTC"))
+        timestamps = [
+          Time.iso8601("2024-03-16T23:54:12Z"),
+          Time.iso8601("2024-03-16T23:54:13Z"),
+          Time.iso8601("2024-03-16T23:54:14Z"),
+        ]
+        values = Arrow::TimestampArray.new(data_type, timestamps)
+        assert_equal(Arrow::Table.new(value: values),
+                     load_csv(<<-CSV, headers: true, timestamp_parsers: [:iso8601]))
+value
+#{timestamps[0].iso8601}
+#{timestamps[1].iso8601}
+#{timestamps[2].iso8601}
+                     CSV
+      end
+
+      test("String") do
+        timestamps = [
+          Time.iso8601("2024-03-16T23:54:12Z"),
+          Time.iso8601("2024-03-16T23:54:13Z"),
+          Time.iso8601("2024-03-16T23:54:14Z"),
+        ]
+        values = Arrow::TimestampArray.new(:second, timestamps)
+        format = "%Y-%m-%dT%H:%M:%S"
+        assert_equal(Arrow::Table.new(value: values).schema,
+                     load_csv(<<-CSV, headers: true, timestamp_parsers: [format]).schema)
+value
+#{timestamps[0].iso8601.chomp("Z")}
+#{timestamps[1].iso8601.chomp("Z")}
+#{timestamps[2].iso8601.chomp("Z")}
+                     CSV
+      end
+    end
   end
 end

data/test/test-data-type.rb

@@ -54,7 +54,8 @@ class DataTypeTest < Test::Unit::TestCase
         "abstract type: <:floating_point>: " +
         "use one of not abstract type: [" +
         "Arrow::DoubleDataType, " +
-        "Arrow::FloatDataType]"
+        "Arrow::FloatDataType, " +
+        "Arrow::HalfFloatDataType]"
       assert_raise(ArgumentError.new(message)) do
         Arrow::DataType.resolve(:floating_point)
       end

data/test/test-dense-union-array.rb

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class DenseUnionArrayTest < Test::Unit::TestCase
+  def setup
+    data_type_fields = [
+      Arrow::Field.new("number", :int16),
+      Arrow::Field.new("text", :string),
+    ]
+    type_codes = [11, 13]
+    @data_type = Arrow::DenseUnionDataType.new(data_type_fields, type_codes)
+    type_ids = Arrow::Int8Array.new([11, 13, 11, 13, 13])
+    value_offsets = Arrow::Int32Array.new([0, 0, 1, 1, 2])
+    fields = [
+      Arrow::Int16Array.new([1, nil]),
+      Arrow::StringArray.new(["a", "b", "c"])
+    ]
+    @array = Arrow::DenseUnionArray.new(@data_type,
+                                        type_ids,
+                                        value_offsets,
+                                        fields)
+  end
+
+  def test_get_value
+    assert_equal([1, "a", nil, "b", "c"],
+                 @array.length.times.collect {|i| @array[i]})
+  end
+end

data/test/test-dense-union-data-type.rb

@@ -35,7 +35,7 @@ class DenseUnionDataTypeTest < Test::Unit::TestCase
     test("description") do
       assert_equal("dense_union<visible: bool=2, count: int32=9>",
                    Arrow::DenseUnionDataType.new(fields: @fields,
-                                                  type_codes: [2, 9]).to_s)
+                                                 type_codes: [2, 9]).to_s)
     end
   end
 end

data/test/test-expression.rb

@@ -36,5 +36,16 @@ class TestExpression < Test::Unit::TestCase
       assert_equal(Arrow::CallExpression.new("func", ["argument1"]),
                    Arrow::Expression.try_convert(["func", "argument1"]))
     end
+
+    test("[Symbol, String, Hash]") do
+      options = Arrow::MatchSubstringOptions.new
+      options.pattern = "hello"
+      assert_equal(Arrow::CallExpression.new("match_substring",
+                                             ["content"],
+                                             options),
+                   Arrow::Expression.try_convert([:match_substring,
+                                                  "content",
+                                                  {pattern: "hello"}]))
+    end
   end
 end

data/test/test-function.rb

@@ -199,12 +199,12 @@ class FunctionTest < Test::Unit::TestCase
   end
 
   def test_call
-      or_function = Arrow::Function.find("or")
-      args = [
-        Arrow::BooleanArray.new([true, false, false]),
-        Arrow::BooleanArray.new([true, false, true]),
-      ]
-      assert_equal([true, false, true],
-                   or_function.call(args).value.to_a)
+    or_function = Arrow::Function.find("or")
+    args = [
+      Arrow::BooleanArray.new([true, false, false]),
+      Arrow::BooleanArray.new([true, false, true]),
+    ]
+    assert_equal([true, false, true],
+                 or_function.call(args).value.to_a)
   end
 end