red-arrow 0.17.1 → 1.0.0

data/test/test-buffer.rb

@@ -16,6 +16,17 @@
 # under the License.
 
 class BufferTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("GC") do
+      data = "Hello"
+      data_id = data.object_id
+      _buffer = Arrow::Buffer.new(data)
+      data = nil
+      GC.start
+      assert_equal("Hello", ObjectSpace._id2ref(data_id))
+    end
+  end
+
   sub_test_case("instance methods") do
     def setup
       @buffer = Arrow::Buffer.new("Hello")

data/test/test-dense-union-data-type.rb

@@ -28,12 +28,12 @@ class DenseUnionDataTypeTest < Test::Unit::TestCase
     end
 
     test("ordered arguments") do
-      assert_equal("union[dense]<visible: bool=2, count: int32=9>",
+      assert_equal("dense_union<visible: bool=2, count: int32=9>",
                    Arrow::DenseUnionDataType.new(@fields, [2, 9]).to_s)
     end
 
     test("description") do
-      assert_equal("union[dense]<visible: bool=2, count: int32=9>",
+      assert_equal("dense_union<visible: bool=2, count: int32=9>",
                    Arrow::DenseUnionDataType.new(fields: @fields,
                                                   type_codes: [2, 9]).to_s)
     end

data/test/test-dictionary-array.rb

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class DictionaryArrayTest < Test::Unit::TestCase
+  sub_test_case("instance methods") do
+    def setup
+      @values = ["a", "b", "c", "b", "a"]
+      @string_array = Arrow::StringArray.new(@values)
+      @array = @string_array.dictionary_encode
+    end
+
+    test("#[]") do
+      assert_equal(@values, @array.to_a)
+    end
+
+    test("#get_value") do
+      assert_equal([
+                     @values[0],
+                     @values[3],
+                   ],
+                   [
+                     @array.get_value(0),
+                     @array.get_value(3),
+                   ])
+    end
+  end
+end

data/test/test-feather.rb

@@ -28,7 +28,7 @@ class FeatherTest < Test::Unit::TestCase
     @output = Tempfile.new(["red-arrow", ".feather"])
     begin
       yield(@output)
-    rescue
+    ensure
       @output.close!
     end
   end

data/test/test-record-batch-iterator.rb

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class RecordBatchIteratorTest < Test::Unit::TestCase
+  def setup
+    @schema = Arrow::Schema.new(visible: :boolean,
+                                count: :uint32)
+    @record_batches = [
+      Arrow::RecordBatch.new(@schema,
+                             visible: [true],
+                             count: [1]),
+      Arrow::RecordBatch.new(@schema,
+                             visible: [false, nil],
+                             count: [nil, 3]),
+    ]
+    @iterator = Arrow::RecordBatchIterator.new(@record_batches)
+  end
+
+  def test_to_a
+    assert_equal(@record_batches,
+                 @iterator.to_a)
+  end
+end

data/test/test-record-batch.rb

@@ -22,6 +22,20 @@ class RecordBatchTest < Test::Unit::TestCase
                                   count: :uint32)
     end
 
+    test("[raw_table]") do
+      raw_table = {
+        visible: [true, nil, false],
+        count: [1, nil, 3],
+      }
+      record_batch = Arrow::RecordBatch.new(raw_table)
+      assert_equal([
+                     {"visible" => true,  "count" => 1},
+                     {"visible" => nil,   "count" => nil},
+                     {"visible" => false, "count" => 3},
+                   ],
+                   record_batch.each_record.collect(&:to_h))
+    end
+
     test("[Schema, records]") do
       records = [
         {visible: true, count: 1},

data/test/test-sparse-union-data-type.rb

@@ -28,12 +28,12 @@ class SparseUnionDataTypeTest < Test::Unit::TestCase
     end
 
     test("ordered arguments") do
-      assert_equal("union[sparse]<visible: bool=2, count: int32=9>",
+      assert_equal("sparse_union<visible: bool=2, count: int32=9>",
                    Arrow::SparseUnionDataType.new(@fields, [2, 9]).to_s)
     end
 
     test("description") do
-      assert_equal("union[sparse]<visible: bool=2, count: int32=9>",
+      assert_equal("sparse_union<visible: bool=2, count: int32=9>",
                    Arrow::SparseUnionDataType.new(fields: @fields,
                                                   type_codes: [2, 9]).to_s)
     end

data/test/test-struct-array-builder.rb

@@ -38,10 +38,12 @@ class StructArrayBuilderTest < Test::Unit::TestCase
 
     test("Array") do
       @builder.append_value([true, 1])
+      @builder.append_value([])
+      @builder.append_value([false])
       array = @builder.finish
       assert_equal([
-                     [true],
-                     [1],
+                     [true, nil, false],
+                     [1, nil, nil],
                    ],
                    [
                      array.find_field(0).to_a,
@@ -66,10 +68,12 @@ class StructArrayBuilderTest < Test::Unit::TestCase
 
     test("Hash") do
       @builder.append_value(count: 1, visible: true)
+      @builder.append_value(visible: false)
+      @builder.append_value(count: 2)
       array = @builder.finish
       assert_equal([
-                     [true],
-                     [1],
+                     [true, false, nil],
+                     [1, nil, 2],
                    ],
                    [
                      array.find_field(0).to_a,

data/test/values/test-dense-union-array.rb

@@ -48,10 +48,7 @@ module ValuesDenseUnionArrayTests
       sub_record_batch.columns[0].data
     end
     values.each do |value|
-      if value.nil?
-        type_ids << nil
-        offsets << 0
-      elsif value.key?("0")
+      if value.key?("0")
         type_id = type_codes[0]
         type_ids << type_id
         offsets << (type_ids.count(type_id) - 1)
@@ -70,7 +67,6 @@ module ValuesDenseUnionArrayTests
   def test_null
     values = [
       {"0" => nil},
-      nil,
     ]
     target = build(:null, values)
     assert_equal(values, target.values)
@@ -79,7 +75,6 @@ module ValuesDenseUnionArrayTests
   def test_boolean
     values = [
       {"0" => true},
-      nil,
       {"1" => nil},
     ]
     target = build(:boolean, values)
@@ -89,7 +84,6 @@ module ValuesDenseUnionArrayTests
   def test_int8
     values = [
       {"0" => -(2 ** 7)},
-      nil,
       {"1" => nil},
     ]
     target = build(:int8, values)
@@ -99,7 +93,6 @@ module ValuesDenseUnionArrayTests
   def test_uint8
     values = [
       {"0" => (2 ** 8) - 1},
-      nil,
       {"1" => nil},
     ]
     target = build(:uint8, values)
@@ -109,7 +102,6 @@ module ValuesDenseUnionArrayTests
   def test_int16
     values = [
       {"0" => -(2 ** 15)},
-      nil,
       {"1" => nil},
     ]
     target = build(:int16, values)
@@ -119,7 +111,6 @@ module ValuesDenseUnionArrayTests
   def test_uint16
     values = [
       {"0" => (2 ** 16) - 1},
-      nil,
       {"1" => nil},
     ]
     target = build(:uint16, values)
@@ -129,7 +120,6 @@ module ValuesDenseUnionArrayTests
   def test_int32
     values = [
       {"0" => -(2 ** 31)},
-      nil,
       {"1" => nil},
     ]
     target = build(:int32, values)
@@ -139,7 +129,6 @@ module ValuesDenseUnionArrayTests
   def test_uint32
     values = [
       {"0" => (2 ** 32) - 1},
-      nil,
       {"1" => nil},
     ]
     target = build(:uint32, values)
@@ -149,7 +138,6 @@ module ValuesDenseUnionArrayTests
   def test_int64
     values = [
       {"0" => -(2 ** 63)},
-      nil,
       {"1" => nil},
     ]
     target = build(:int64, values)
@@ -159,7 +147,6 @@ module ValuesDenseUnionArrayTests
   def test_uint64
     values = [
       {"0" => (2 ** 64) - 1},
-      nil,
       {"1" => nil},
     ]
     target = build(:uint64, values)
@@ -169,7 +156,6 @@ module ValuesDenseUnionArrayTests
   def test_float
     values = [
       {"0" => -1.0},
-      nil,
       {"1" => nil},
     ]
     target = build(:float, values)
@@ -179,7 +165,6 @@ module ValuesDenseUnionArrayTests
   def test_double
     values = [
       {"0" => -1.0},
-      nil,
       {"1" => nil},
     ]
     target = build(:double, values)
@@ -189,7 +174,6 @@ module ValuesDenseUnionArrayTests
   def test_binary
     values = [
       {"0" => "\xff".b},
-      nil,
       {"1" => nil},
     ]
     target = build(:binary, values)
@@ -199,7 +183,6 @@ module ValuesDenseUnionArrayTests
   def test_string
     values = [
       {"0" => "Ruby"},
-      nil,
       {"1" => nil},
     ]
     target = build(:string, values)
@@ -209,7 +192,6 @@ module ValuesDenseUnionArrayTests
   def test_date32
     values = [
       {"0" => Date.new(1960, 1, 1)},
-      nil,
       {"1" => nil},
     ]
     target = build(:date32, values)
@@ -219,7 +201,6 @@ module ValuesDenseUnionArrayTests
   def test_date64
     values = [
       {"0" => DateTime.new(1960, 1, 1, 2, 9, 30)},
-      nil,
       {"1" => nil},
     ]
     target = build(:date64, values)
@@ -229,7 +210,6 @@ module ValuesDenseUnionArrayTests
   def test_timestamp_second
     values = [
       {"0" => Time.parse("1960-01-01T02:09:30Z")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -243,7 +223,6 @@ module ValuesDenseUnionArrayTests
   def test_timestamp_milli
     values = [
       {"0" => Time.parse("1960-01-01T02:09:30.123Z")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -257,7 +236,6 @@ module ValuesDenseUnionArrayTests
   def test_timestamp_micro
     values = [
       {"0" => Time.parse("1960-01-01T02:09:30.123456Z")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -271,7 +249,6 @@ module ValuesDenseUnionArrayTests
   def test_timestamp_nano
     values = [
       {"0" => Time.parse("1960-01-01T02:09:30.123456789Z")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -287,7 +264,6 @@ module ValuesDenseUnionArrayTests
     values = [
       # 00:10:00
       {"0" => Arrow::Time.new(unit, 60 * 10)},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -303,7 +279,6 @@ module ValuesDenseUnionArrayTests
     values = [
       # 00:10:00.123
       {"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -319,7 +294,6 @@ module ValuesDenseUnionArrayTests
     values = [
       # 00:10:00.123456
       {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -335,7 +309,6 @@ module ValuesDenseUnionArrayTests
     values = [
       # 00:10:00.123456789
       {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -349,7 +322,6 @@ module ValuesDenseUnionArrayTests
   def test_decimal128
     values = [
       {"0" => BigDecimal("92.92")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -364,7 +336,6 @@ module ValuesDenseUnionArrayTests
   def test_list
     values = [
       {"0" => [true, nil, false]},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -381,7 +352,6 @@ module ValuesDenseUnionArrayTests
   def test_struct
     values = [
       {"0" => {"sub_field" => true}},
-      nil,
       {"1" => nil},
       {"0" => {"sub_field" => nil}},
     ]
@@ -402,7 +372,6 @@ module ValuesDenseUnionArrayTests
     omit("Need to add support for SparseUnionArrayBuilder")
     values = [
       {"0" => {"field1" => true}},
-      nil,
       {"1" => nil},
       {"0" => {"field2" => nil}},
     ]
@@ -428,7 +397,6 @@ module ValuesDenseUnionArrayTests
     omit("Need to add support for DenseUnionArrayBuilder")
     values = [
       {"0" => {"field1" => true}},
-      nil,
       {"1" => nil},
       {"0" => {"field2" => nil}},
     ]
@@ -454,7 +422,6 @@ module ValuesDenseUnionArrayTests
     omit("Need to add support for DictionaryArrayBuilder")
     values = [
       {"0" => "Ruby"},
-      nil,
       {"1" => nil},
       {"0" => "GLib"},
     ]

data/test/values/test-sparse-union-array.rb

@@ -44,9 +44,7 @@ module ValuesSparseUnionArrayTests
       sub_record_batch.columns[0].data
     end
     values.each do |value|
-      if value.nil?
-        type_ids << nil
-      elsif value.key?("0")
+      if value.key?("0")
         type_ids << type_codes[0]
       elsif value.key?("1")
         type_ids << type_codes[1]
@@ -60,7 +58,6 @@ module ValuesSparseUnionArrayTests
   def test_null
     values = [
       {"0" => nil},
-      nil,
     ]
     target = build(:null, values)
     assert_equal(values, target.values)
@@ -69,7 +66,6 @@ module ValuesSparseUnionArrayTests
   def test_boolean
     values = [
       {"0" => true},
-      nil,
       {"1" => nil},
     ]
     target = build(:boolean, values)
@@ -79,7 +75,6 @@ module ValuesSparseUnionArrayTests
   def test_int8
     values = [
       {"0" => -(2 ** 7)},
-      nil,
       {"1" => nil},
     ]
     target = build(:int8, values)
@@ -89,7 +84,6 @@ module ValuesSparseUnionArrayTests
   def test_uint8
     values = [
       {"0" => (2 ** 8) - 1},
-      nil,
       {"1" => nil},
     ]
     target = build(:uint8, values)
@@ -99,7 +93,6 @@ module ValuesSparseUnionArrayTests
   def test_int16
     values = [
       {"0" => -(2 ** 15)},
-      nil,
       {"1" => nil},
     ]
     target = build(:int16, values)
@@ -109,7 +102,6 @@ module ValuesSparseUnionArrayTests
   def test_uint16
     values = [
       {"0" => (2 ** 16) - 1},
-      nil,
       {"1" => nil},
     ]
     target = build(:uint16, values)
@@ -119,7 +111,6 @@ module ValuesSparseUnionArrayTests
   def test_int32
     values = [
       {"0" => -(2 ** 31)},
-      nil,
       {"1" => nil},
     ]
     target = build(:int32, values)
@@ -129,7 +120,6 @@ module ValuesSparseUnionArrayTests
   def test_uint32
     values = [
       {"0" => (2 ** 32) - 1},
-      nil,
       {"1" => nil},
     ]
     target = build(:uint32, values)
@@ -139,7 +129,6 @@ module ValuesSparseUnionArrayTests
   def test_int64
     values = [
       {"0" => -(2 ** 63)},
-      nil,
       {"1" => nil},
     ]
     target = build(:int64, values)
@@ -149,7 +138,6 @@ module ValuesSparseUnionArrayTests
   def test_uint64
     values = [
       {"0" => (2 ** 64) - 1},
-      nil,
       {"1" => nil},
     ]
     target = build(:uint64, values)
@@ -159,7 +147,6 @@ module ValuesSparseUnionArrayTests
   def test_float
     values = [
       {"0" => -1.0},
-      nil,
       {"1" => nil},
     ]
     target = build(:float, values)
@@ -169,7 +156,6 @@ module ValuesSparseUnionArrayTests
   def test_double
     values = [
       {"0" => -1.0},
-      nil,
       {"1" => nil},
     ]
     target = build(:double, values)
@@ -179,7 +165,6 @@ module ValuesSparseUnionArrayTests
   def test_binary
     values = [
       {"0" => "\xff".b},
-      nil,
       {"1" => nil},
     ]
     target = build(:binary, values)
@@ -189,7 +174,6 @@ module ValuesSparseUnionArrayTests
   def test_string
     values = [
       {"0" => "Ruby"},
-      nil,
       {"1" => nil},
     ]
     target = build(:string, values)
@@ -199,7 +183,6 @@ module ValuesSparseUnionArrayTests
   def test_date32
     values = [
       {"0" => Date.new(1960, 1, 1)},
-      nil,
       {"1" => nil},
     ]
     target = build(:date32, values)
@@ -209,7 +192,6 @@ module ValuesSparseUnionArrayTests
   def test_date64
     values = [
       {"0" => DateTime.new(1960, 1, 1, 2, 9, 30)},
-      nil,
       {"1" => nil},
     ]
     target = build(:date64, values)
@@ -219,7 +201,6 @@ module ValuesSparseUnionArrayTests
   def test_timestamp_second
     values = [
       {"0" => Time.parse("1960-01-01T02:09:30Z")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -233,7 +214,6 @@ module ValuesSparseUnionArrayTests
   def test_timestamp_milli
     values = [
       {"0" => Time.parse("1960-01-01T02:09:30.123Z")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -247,7 +227,6 @@ module ValuesSparseUnionArrayTests
   def test_timestamp_micro
     values = [
       {"0" => Time.parse("1960-01-01T02:09:30.123456Z")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -261,7 +240,6 @@ module ValuesSparseUnionArrayTests
   def test_timestamp_nano
     values = [
       {"0" => Time.parse("1960-01-01T02:09:30.123456789Z")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -277,7 +255,6 @@ module ValuesSparseUnionArrayTests
     values = [
       # 00:10:00
       {"0" => Arrow::Time.new(unit, 60 * 10)},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -293,7 +270,6 @@ module ValuesSparseUnionArrayTests
     values = [
       # 00:10:00.123
       {"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -309,7 +285,6 @@ module ValuesSparseUnionArrayTests
     values = [
       # 00:10:00.123456
       {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -325,7 +300,6 @@ module ValuesSparseUnionArrayTests
     values = [
       # 00:10:00.123456789
       {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -339,7 +313,6 @@ module ValuesSparseUnionArrayTests
   def test_decimal128
     values = [
       {"0" => BigDecimal("92.92")},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -354,7 +327,6 @@ module ValuesSparseUnionArrayTests
   def test_list
     values = [
       {"0" => [true, nil, false]},
-      nil,
       {"1" => nil},
     ]
     target = build({
@@ -371,7 +343,6 @@ module ValuesSparseUnionArrayTests
   def test_struct
     values = [
       {"0" => {"sub_field" => true}},
-      nil,
       {"1" => nil},
       {"0" => {"sub_field" => nil}},
     ]
@@ -392,7 +363,6 @@ module ValuesSparseUnionArrayTests
     omit("Need to add support for SparseUnionArrayBuilder")
     values = [
       {"0" => {"field1" => true}},
-      nil,
       {"1" => nil},
       {"0" => {"field2" => nil}},
     ]
@@ -418,7 +388,6 @@ module ValuesSparseUnionArrayTests
     omit("Need to add support for DenseUnionArrayBuilder")
     values = [
       {"0" => {"field1" => true}},
-      nil,
       {"1" => nil},
       {"0" => {"field2" => nil}},
     ]
@@ -444,7 +413,6 @@ module ValuesSparseUnionArrayTests
     omit("Need to add support for DictionaryArrayBuilder")
     values = [
       {"0" => "Ruby"},
-      nil,
       {"1" => nil},
       {"0" => "GLib"},
     ]