red-arrow 11.0.0 → 12.0.0

data/test/raw-records/test-list-array.rb

@@ -509,13 +509,31 @@ module RawRecordsListArrayTests
     assert_equal(records, target.raw_records)
   end
 
-  def test_sparse
-    omit("Need to add support for SparseUnionArrayBuilder")
+  def remove_union_field_names(records)
+    records.collect do |record|
+      record.collect do |column|
+        if column.nil?
+          column
+        else
+          column.collect do |value|
+            if value.nil?
+              value
+            else
+              value.values[0]
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def test_sparse_union
     records = [
       [
         [
           {"field1" => true},
           nil,
+          {"field2" => 29},
           {"field2" => nil},
         ],
       ],
@@ -536,16 +554,17 @@ module RawRecordsListArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_union_field_names(records),
+                 target.raw_records)
   end
 
-  def test_dense
-    omit("Need to add support for DenseUnionArrayBuilder")
+  def test_dense_union
     records = [
       [
         [
           {"field1" => true},
           nil,
+          {"field2" => 29},
           {"field2" => nil},
         ],
       ],
@@ -566,11 +585,11 @@ module RawRecordsListArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_union_field_names(records),
+                 target.raw_records)
   end
 
   def test_dictionary
-    omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [
         [
@@ -581,12 +600,11 @@ module RawRecordsListArrayTests
       ],
       [nil],
     ]
-    dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
     target = build({
                      type: :dictionary,
                      index_data_type: :int8,
-                     dictionary: dictionary,
-                     ordered: true,
+                     value_data_type: :string,
+                     ordered: false,
                    },
                    records)
     assert_equal(records, target.raw_records)

data/test/raw-records/test-map-array.rb

@@ -395,10 +395,33 @@ module RawRecordsMapArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def remove_union_field_names(records)
+    records.collect do |record|
+      record.collect do |column|
+        if column.nil?
+          column
+        else
+          value = {}
+          column.each do |k, v|
+            v = v.values[0] unless v.nil?
+            value[k] = v
+          end
+          value
+        end
+      end
+    end
+  end
+
   def test_sparse_union
-    omit("Need to add support for SparseUnionArrayBuilder")
     records = [
-      [{"key1" => {"field" => true, "key2" => nil, "key3" => {"field" => nil}}}],
+      [
+        {
+          "key1" => {"field1" => true},
+          "key2" => nil,
+          "key3" => {"field2" => 29},
+          "key4" => {"field2" => nil},
+        },
+      ],
       [nil],
     ]
     target = build({
@@ -416,13 +439,20 @@ module RawRecordsMapArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_union_field_names(records),
+                 target.raw_records)
   end
 
   def test_dense_union
-    omit("Need to add support for DenseUnionArrayBuilder")
     records = [
-      [{"key1" => {"field1" => true}, "key2" => nil, "key3" => {"field2" => nil}}],
+      [
+        {
+          "key1" => {"field1" => true},
+          "key2" => nil,
+          "key3" => {"field2" => 29},
+          "key4" => {"field2" => nil},
+        },
+      ],
       [nil],
     ]
     target = build({
@@ -440,21 +470,20 @@ module RawRecordsMapArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_union_field_names(records),
+                 target.raw_records)
   end
 
   def test_dictionary
-    omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [{"key1" => "Ruby", "key2" => nil, "key3" => "GLib"}],
       [nil],
     ]
-    dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
     target = build({
                      type: :dictionary,
                      index_data_type: :int8,
-                     dictionary: dictionary,
-                     ordered: true,
+                     value_data_type: :string,
+                     ordered: false,
                    },
                    records)
     assert_equal(records, target.raw_records)

data/test/raw-records/test-sparse-union-array.rb

@@ -66,12 +66,25 @@ module RawRecordsSparseUnionArrayTests
                            [union_array])
   end
 
+  def remove_field_names(records)
+    records.collect do |record|
+      record.collect do |column|
+        if column.nil?
+          column
+        else
+          column.values[0]
+        end
+      end
+    end
+  end
+
   def test_null
     records = [
       [{"0" => nil}],
     ]
     target = build(:null, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_boolean
@@ -80,7 +93,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:boolean, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_int8
@@ -89,7 +103,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:int8, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_uint8
@@ -98,7 +113,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:uint8, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_int16
@@ -107,7 +123,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:int16, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_uint16
@@ -116,7 +133,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:uint16, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_int32
@@ -125,7 +143,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:int32, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_uint32
@@ -134,7 +153,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:uint32, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_int64
@@ -143,7 +163,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:int64, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_uint64
@@ -152,7 +173,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:uint64, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_float
@@ -161,7 +183,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:float, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_double
@@ -170,7 +193,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:double, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_binary
@@ -179,7 +203,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:binary, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_string
@@ -188,7 +213,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:string, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_date32
@@ -197,7 +223,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:date32, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_date64
@@ -206,7 +233,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:date64, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_timestamp_second
@@ -219,7 +247,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :second,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_timestamp_milli
@@ -232,7 +261,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :milli,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_timestamp_micro
@@ -245,7 +275,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :micro,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_timestamp_nano
@@ -258,7 +289,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :nano,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_time32_second
@@ -273,7 +305,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :second,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_time32_milli
@@ -288,7 +321,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :milli,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_time64_micro
@@ -303,7 +337,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :micro,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_time64_nano
@@ -318,7 +353,8 @@ module RawRecordsSparseUnionArrayTests
                      unit: :nano,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_decimal128
@@ -332,7 +368,8 @@ module RawRecordsSparseUnionArrayTests
                      scale: 2,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_decimal256
@@ -346,7 +383,8 @@ module RawRecordsSparseUnionArrayTests
                      scale: 2,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_month_interval
@@ -355,7 +393,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:month_interval, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_day_time_interval
@@ -364,7 +403,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:day_time_interval, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_month_day_nano_interval
@@ -373,7 +413,8 @@ module RawRecordsSparseUnionArrayTests
       [{"1" => nil}],
     ]
     target = build(:month_day_nano_interval, records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_list
@@ -389,7 +430,8 @@ module RawRecordsSparseUnionArrayTests
                      },
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_struct
@@ -408,7 +450,8 @@ module RawRecordsSparseUnionArrayTests
                      ],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_map
@@ -422,14 +465,15 @@ module RawRecordsSparseUnionArrayTests
                      item: :boolean,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 
   def test_sparse_union
-    omit("Need to add support for SparseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
       [{"1" => nil}],
+      [{"0" => {"field2" => 29}}],
       [{"0" => {"field2" => nil}}],
     ]
     target = build({
@@ -447,14 +491,15 @@ module RawRecordsSparseUnionArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(remove_field_names(records)),
+                 target.raw_records)
   end
 
   def test_dense_union
-    omit("Need to add support for DenseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
       [{"1" => nil}],
+      [{"0" => {"field2" => 29}}],
       [{"0" => {"field2" => nil}}],
     ]
     target = build({
@@ -472,25 +517,25 @@ module RawRecordsSparseUnionArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(remove_field_names(records)),
+                 target.raw_records)
   end
 
   def test_dictionary
-    omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [{"0" => "Ruby"}],
       [{"1" => nil}],
       [{"0" => "GLib"}],
     ]
-    dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
     target = build({
                      type: :dictionary,
                      index_data_type: :int8,
-                     dictionary: dictionary,
-                     ordered: true,
+                     value_data_type: :string,
+                     ordered: false,
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_field_names(records),
+                 target.raw_records)
   end
 end
 

data/test/raw-records/test-struct-array.rb

@@ -426,12 +426,26 @@ module RawRecordsStructArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def remove_union_field_names(records)
+    records.collect do |record|
+      record.collect do |column|
+        if column.nil?
+          column
+        else
+          value = column["field"]
+          value = value.values[0] unless value.nil?
+          {"field" => value}
+        end
+      end
+    end
+  end
+
   def test_sparse_union
-    omit("Need to add support for SparseUnionArrayBuilder")
     records = [
       [{"field" => {"field1" => true}}],
       [nil],
       [{"field" => nil}],
+      [{"field" => {"field2" => 29}}],
       [{"field" => {"field2" => nil}}],
     ]
     target = build({
@@ -449,15 +463,16 @@ module RawRecordsStructArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_union_field_names(records),
+                 target.raw_records)
   end
 
   def test_dense_union
-    omit("Need to add support for DenseUnionArrayBuilder")
     records = [
       [{"field" => {"field1" => true}}],
       [nil],
       [{"field" => nil}],
+      [{"field" => {"field2" => 29}}],
       [{"field" => {"field2" => nil}}],
     ]
     target = build({
@@ -475,23 +490,22 @@ module RawRecordsStructArrayTests
                      type_codes: [0, 1],
                    },
                    records)
-    assert_equal(records, target.raw_records)
+    assert_equal(remove_union_field_names(records),
+                 target.raw_records)
   end
 
   def test_dictionary
-    omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [{"field" => "Ruby"}],
       [nil],
       [{"field" => nil}],
       [{"field" => "GLib"}],
     ]
-    dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
     target = build({
                      type: :dictionary,
                      index_data_type: :int8,
-                     dictionary: dictionary,
-                     ordered: true,
+                     value_data_type: :string,
+                     ordered: false,
                    },
                    records)
     assert_equal(records, target.raw_records)

data/test/test-array.rb

@@ -322,4 +322,11 @@ class ArrayTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case("#index") do
+    test("Integer") do
+      assert_equal(2,
+                   Arrow::Int32Array.new([1, 2, 3, 4, 5]).index(3))
+    end
+  end
 end

data/test/test-chunked-array.rb

@@ -186,4 +186,13 @@ class ChunkedArrayTest < Test::Unit::TestCase
     assert_equal(Arrow::ChunkedArray.new([["1", nil, "3"]]),
                  chunked_array.cast(:string))
   end
+
+  test("#index") do
+    arrays = [
+      Arrow::Int32Array.new([1, 2]),
+      Arrow::Int32Array.new([3, 4, 5]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(arrays)
+    assert_equal(2, chunked_array.index(3))
+  end
 end

data/test/test-dense-union-array.rb

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class DenseUnionArrayTest < Test::Unit::TestCase
+  def setup
+    data_type_fields = [
+      Arrow::Field.new("number", :int16),
+      Arrow::Field.new("text", :string),
+    ]
+    type_codes = [11, 13]
+    @data_type = Arrow::DenseUnionDataType.new(data_type_fields, type_codes)
+    type_ids = Arrow::Int8Array.new([11, 13, 11, 13, 13])
+    value_offsets = Arrow::Int32Array.new([0, 0, 1, 1, 2])
+    fields = [
+      Arrow::Int16Array.new([1, nil]),
+      Arrow::StringArray.new(["a", "b", "c"])
+    ]
+    @array = Arrow::DenseUnionArray.new(@data_type,
+                                        type_ids,
+                                        value_offsets,
+                                        fields)
+  end
+
+  def test_get_value
+    assert_equal([1, "a", nil, "b", "c"],
+                 @array.length.times.collect {|i| @array[i]})
+  end
+end