red-arrow 0.16.0 → 2.0.0

data/test/raw-records/test-sparse-union-array.rb

@@ -51,9 +51,7 @@ module RawRecordsSparseUnionArrayTests
     end
     records.each do |record|
       column = record[0]
-      if column.nil?
-        type_ids << nil
-      elsif column.key?("0")
+      if column.key?("0")
         type_ids << type_codes[0]
       elsif column.key?("1")
         type_ids << type_codes[1]
@@ -71,7 +69,6 @@ module RawRecordsSparseUnionArrayTests
   def test_null
     records = [
       [{"0" => nil}],
-      [nil],
     ]
     target = build(:null, records)
     assert_equal(records, target.raw_records)
@@ -80,7 +77,6 @@ module RawRecordsSparseUnionArrayTests
   def test_boolean
     records = [
       [{"0" => true}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:boolean, records)
@@ -90,7 +86,6 @@ module RawRecordsSparseUnionArrayTests
   def test_int8
     records = [
       [{"0" => -(2 ** 7)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int8, records)
@@ -100,7 +95,6 @@ module RawRecordsSparseUnionArrayTests
   def test_uint8
     records = [
       [{"0" => (2 ** 8) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint8, records)
@@ -110,7 +104,6 @@ module RawRecordsSparseUnionArrayTests
   def test_int16
     records = [
       [{"0" => -(2 ** 15)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int16, records)
@@ -120,7 +113,6 @@ module RawRecordsSparseUnionArrayTests
   def test_uint16
     records = [
       [{"0" => (2 ** 16) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint16, records)
@@ -130,7 +122,6 @@ module RawRecordsSparseUnionArrayTests
   def test_int32
     records = [
       [{"0" => -(2 ** 31)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int32, records)
@@ -140,7 +131,6 @@ module RawRecordsSparseUnionArrayTests
   def test_uint32
     records = [
       [{"0" => (2 ** 32) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint32, records)
@@ -150,7 +140,6 @@ module RawRecordsSparseUnionArrayTests
   def test_int64
     records = [
       [{"0" => -(2 ** 63)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int64, records)
@@ -160,7 +149,6 @@ module RawRecordsSparseUnionArrayTests
   def test_uint64
     records = [
       [{"0" => (2 ** 64) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint64, records)
@@ -170,7 +158,6 @@ module RawRecordsSparseUnionArrayTests
   def test_float
     records = [
       [{"0" => -1.0}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:float, records)
@@ -180,7 +167,6 @@ module RawRecordsSparseUnionArrayTests
   def test_double
     records = [
       [{"0" => -1.0}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:double, records)
@@ -190,7 +176,6 @@ module RawRecordsSparseUnionArrayTests
   def test_binary
     records = [
       [{"0" => "\xff".b}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:binary, records)
@@ -200,7 +185,6 @@ module RawRecordsSparseUnionArrayTests
   def test_string
     records = [
       [{"0" => "Ruby"}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:string, records)
@@ -210,7 +194,6 @@ module RawRecordsSparseUnionArrayTests
   def test_date32
     records = [
       [{"0" => Date.new(1960, 1, 1)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:date32, records)
@@ -220,7 +203,6 @@ module RawRecordsSparseUnionArrayTests
   def test_date64
     records = [
       [{"0" => DateTime.new(1960, 1, 1, 2, 9, 30)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:date64, records)
@@ -230,7 +212,6 @@ module RawRecordsSparseUnionArrayTests
   def test_timestamp_second
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -244,7 +225,6 @@ module RawRecordsSparseUnionArrayTests
   def test_timestamp_milli
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -258,7 +238,6 @@ module RawRecordsSparseUnionArrayTests
   def test_timestamp_micro
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123456Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -272,7 +251,6 @@ module RawRecordsSparseUnionArrayTests
   def test_timestamp_nano
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123456789Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -288,7 +266,6 @@ module RawRecordsSparseUnionArrayTests
     records = [
       # 00:10:00
       [{"0" => Arrow::Time.new(unit, 60 * 10)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -304,7 +281,6 @@ module RawRecordsSparseUnionArrayTests
     records = [
       # 00:10:00.123
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -320,7 +296,6 @@ module RawRecordsSparseUnionArrayTests
     records = [
       # 00:10:00.123456
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -336,7 +311,6 @@ module RawRecordsSparseUnionArrayTests
     records = [
       # 00:10:00.123456789
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -350,7 +324,6 @@ module RawRecordsSparseUnionArrayTests
   def test_decimal128
     records = [
       [{"0" => BigDecimal("92.92")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -365,7 +338,6 @@ module RawRecordsSparseUnionArrayTests
   def test_list
     records = [
       [{"0" => [true, nil, false]}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -382,7 +354,6 @@ module RawRecordsSparseUnionArrayTests
   def test_struct
     records = [
       [{"0" => {"sub_field" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"sub_field" => nil}}],
     ]
@@ -403,7 +374,6 @@ module RawRecordsSparseUnionArrayTests
     omit("Need to add support for SparseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"field2" => nil}}],
     ]
@@ -429,7 +399,6 @@ module RawRecordsSparseUnionArrayTests
     omit("Need to add support for DenseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"field2" => nil}}],
     ]
@@ -455,7 +424,6 @@ module RawRecordsSparseUnionArrayTests
     omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [{"0" => "Ruby"}],
-      [nil],
       [{"1" => nil}],
       [{"0" => "GLib"}],
     ]

data/test/run-test.rb

@@ -19,6 +19,7 @@
 
 $VERBOSE = true
 
+require "fileutils"
 require "pathname"
 
 (ENV["ARROW_DLL_PATH"] || "").split(File::PATH_SEPARATOR).each do |path|
@@ -31,6 +32,14 @@ lib_dir = base_dir + "lib"
 ext_dir = base_dir + "ext" + "arrow"
 test_dir = base_dir + "test"
 
+build_dir = ENV["BUILD_DIR"]
+if build_dir
+  build_dir = File.join(build_dir, "red-arrow")
+  FileUtils.mkdir_p(build_dir)
+else
+  build_dir = ext_dir
+end
+
 make = nil
 if ENV["NO_MAKE"] != "yes"
   if ENV["MAKE"]
@@ -42,15 +51,17 @@ if ENV["NO_MAKE"] != "yes"
   end
 end
 if make
-  Dir.chdir(ext_dir.to_s) do
+  Dir.chdir(build_dir.to_s) do
     unless File.exist?("Makefile")
-      system(RbConfig.ruby, "extconf.rb", "--enable-debug-build") or exit(false)
+      system(RbConfig.ruby,
+             (ext_dir + "extconf.rb").to_s,
+             "--enable-debug-build") or exit(false)
     end
     system("#{make} > #{File::NULL}") or exit(false)
   end
 end
 
-$LOAD_PATH.unshift(ext_dir.to_s)
+$LOAD_PATH.unshift(build_dir.to_s)
 $LOAD_PATH.unshift(lib_dir.to_s)
 
 require_relative "helper"

data/test/test-array.rb

@@ -76,20 +76,22 @@ class ArrayTest < Test::Unit::TestCase
     def setup
       values = [true, false, false, true]
       @array = Arrow::BooleanArray.new(values)
+      @options = Arrow::FilterOptions.new
+      @options.null_selection_behavior = :emit_null
     end
 
     test("Array: boolean") do
       filter = [nil, true, true, false]
       filtered_array = Arrow::BooleanArray.new([nil, false, false])
       assert_equal(filtered_array,
-                   @array.filter(filter))
+                   @array.filter(filter, @options))
     end
 
     test("Arrow::BooleanArray") do
       filter = Arrow::BooleanArray.new([nil, true, true, false])
       filtered_array = Arrow::BooleanArray.new([nil, false, false])
       assert_equal(filtered_array,
-                   @array.filter(filter))
+                   @array.filter(filter, @options))
     end
 
     test("Arrow::ChunkedArray") do
@@ -100,7 +102,7 @@ class ArrayTest < Test::Unit::TestCase
       filter = Arrow::ChunkedArray.new(chunks)
       filtered_array = Arrow::BooleanArray.new([nil, false, false])
       assert_equal(filtered_array,
-                   @array.filter(filter))
+                   @array.filter(filter, @options))
     end
   end
 

data/test/test-buffer.rb

@@ -16,6 +16,17 @@
 # under the License.
 
 class BufferTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("GC") do
+      data = "Hello"
+      data_id = data.object_id
+      _buffer = Arrow::Buffer.new(data)
+      data = nil
+      GC.start
+      assert_equal("Hello", ObjectSpace._id2ref(data_id))
+    end
+  end
+
   sub_test_case("instance methods") do
     def setup
       @buffer = Arrow::Buffer.new("Hello")

data/test/test-chunked-array.rb

@@ -92,6 +92,8 @@ class ChunkedArrayTest < Test::Unit::TestCase
         Arrow::BooleanArray.new([false, true, false]),
       ]
       @chunked_array = Arrow::ChunkedArray.new(arrays)
+      @options = Arrow::FilterOptions.new
+      @options.null_selection_behavior = :emit_null
     end
 
     test("Array: boolean") do
@@ -102,7 +104,7 @@ class ChunkedArrayTest < Test::Unit::TestCase
       ]
       filtered_chunked_array = Arrow::ChunkedArray.new(chunks)
       assert_equal(filtered_chunked_array,
-                   @chunked_array.filter(filter))
+                   @chunked_array.filter(filter, @options))
     end
 
     test("Arrow::BooleanArray") do
@@ -113,7 +115,7 @@ class ChunkedArrayTest < Test::Unit::TestCase
       ]
       filtered_chunked_array = Arrow::ChunkedArray.new(chunks)
       assert_equal(filtered_chunked_array,
-                   @chunked_array.filter(filter))
+                   @chunked_array.filter(filter, @options))
     end
 
     test("Arrow::ChunkedArray") do
@@ -128,7 +130,7 @@ class ChunkedArrayTest < Test::Unit::TestCase
       ]
       filtered_chunked_array = Arrow::ChunkedArray.new(filtered_chunks)
       assert_equal(filtered_chunked_array,
-                   @chunked_array.filter(filter))
+                   @chunked_array.filter(filter, @options))
     end
   end
 

data/test/test-dense-union-data-type.rb

@@ -28,12 +28,12 @@ class DenseUnionDataTypeTest < Test::Unit::TestCase
     end
 
     test("ordered arguments") do
-      assert_equal("union[dense]<visible: bool=2, count: int32=9>",
+      assert_equal("dense_union<visible: bool=2, count: int32=9>",
                    Arrow::DenseUnionDataType.new(@fields, [2, 9]).to_s)
     end
 
     test("description") do
-      assert_equal("union[dense]<visible: bool=2, count: int32=9>",
+      assert_equal("dense_union<visible: bool=2, count: int32=9>",
                    Arrow::DenseUnionDataType.new(fields: @fields,
                                                   type_codes: [2, 9]).to_s)
     end

data/test/test-dictionary-array.rb

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class DictionaryArrayTest < Test::Unit::TestCase
+  sub_test_case("instance methods") do
+    def setup
+      @values = ["a", "b", "c", "b", "a"]
+      @string_array = Arrow::StringArray.new(@values)
+      @array = @string_array.dictionary_encode
+    end
+
+    test("#[]") do
+      assert_equal(@values, @array.to_a)
+    end
+
+    test("#get_value") do
+      assert_equal([
+                     @values[0],
+                     @values[3],
+                   ],
+                   [
+                     @array.get_value(0),
+                     @array.get_value(3),
+                   ])
+    end
+  end
+end

data/test/test-feather.rb

@@ -18,17 +18,32 @@
 class FeatherTest < Test::Unit::TestCase
   include Helper::Fixture
 
-  def test_save_load
+  def setup
     columns = {
       "message" => Arrow::StringArray.new(["Start", "Crash", "Shutdown"]),
       "is_critical" => Arrow::BooleanArray.new([false, true, false]),
     }
-    table = Arrow::Table.new(columns)
+    @table = Arrow::Table.new(columns)
 
-    output = Tempfile.new(["red-arrow", ".feather"])
-    table.save(output.path)
-    output.close
+    @output = Tempfile.new(["red-arrow", ".feather"])
+    begin
+      yield(@output)
+    ensure
+      @output.close!
+    end
+  end
+
+  def test_default
+    @table.save(@output.path)
+    @output.close
+
+    assert_equal(@table, Arrow::Table.load(@output.path))
+  end
+
+  def test_compression
+    @table.save(@output.path, compression: :zstd)
+    @output.close
 
-    assert_equal(table, Arrow::Table.load(output.path))
+    assert_equal(@table, Arrow::Table.load(@output.path))
   end
 end