red-arrow 0.15.1 → 1.0.1

data/test/run-test.rb

@@ -19,6 +19,7 @@
 
 $VERBOSE = true
 
+require "fileutils"
 require "pathname"
 
 (ENV["ARROW_DLL_PATH"] || "").split(File::PATH_SEPARATOR).each do |path|
@@ -31,6 +32,14 @@ lib_dir = base_dir + "lib"
 ext_dir = base_dir + "ext" + "arrow"
 test_dir = base_dir + "test"
 
+build_dir = ENV["BUILD_DIR"]
+if build_dir
+  build_dir = File.join(build_dir, "red-arrow")
+  FileUtils.mkdir_p(build_dir)
+else
+  build_dir = ext_dir
+end
+
 make = nil
 if ENV["NO_MAKE"] != "yes"
   if ENV["MAKE"]
@@ -42,15 +51,17 @@ if ENV["NO_MAKE"] != "yes"
   end
 end
 if make
-  Dir.chdir(ext_dir.to_s) do
+  Dir.chdir(build_dir.to_s) do
     unless File.exist?("Makefile")
-      system(RbConfig.ruby, "extconf.rb", "--enable-debug-build") or exit(false)
+      system(RbConfig.ruby,
+             (ext_dir + "extconf.rb").to_s,
+             "--enable-debug-build") or exit(false)
     end
     system("#{make} > #{File::NULL}") or exit(false)
   end
 end
 
-$LOAD_PATH.unshift(ext_dir.to_s)
+$LOAD_PATH.unshift(build_dir.to_s)
 $LOAD_PATH.unshift(lib_dir.to_s)
 
 require_relative "helper"

data/test/test-array-builder.rb

@@ -60,6 +60,23 @@ class ArrayBuilderTest < Test::Unit::TestCase
                        DateTime.new(2018, 1, 5, 0, 23, 21),
                      ])
       end
+
+      test("list<boolean>s") do
+        assert_build(Arrow::ArrayBuilder,
+                     [
+                       [nil, true, false],
+                       nil,
+                       [false],
+                     ])
+      end
+
+      test("list<string>s") do
+        assert_build(Arrow::ArrayBuilder,
+                     [
+                       ["Hello", "World"],
+                       ["Apache Arrow"],
+                     ])
+      end
     end
 
     sub_test_case("specific builder") do

data/test/test-array.rb

@@ -63,5 +63,109 @@ class ArrayTest < Test::Unit::TestCase
         end
       end
     end
+
+    sub_test_case("#cast") do
+      test("Symbol") do
+        assert_equal(Arrow::Int32Array.new([1, 2, 3]),
+                     Arrow::StringArray.new(["1", "2", "3"]).cast(:int32))
+      end
+    end
+  end
+
+  sub_test_case("#filter") do
+    def setup
+      values = [true, false, false, true]
+      @array = Arrow::BooleanArray.new(values)
+      @options = Arrow::FilterOptions.new
+      @options.null_selection_behavior = :emit_null
+    end
+
+    test("Array: boolean") do
+      filter = [nil, true, true, false]
+      filtered_array = Arrow::BooleanArray.new([nil, false, false])
+      assert_equal(filtered_array,
+                   @array.filter(filter, @options))
+    end
+
+    test("Arrow::BooleanArray") do
+      filter = Arrow::BooleanArray.new([nil, true, true, false])
+      filtered_array = Arrow::BooleanArray.new([nil, false, false])
+      assert_equal(filtered_array,
+                   @array.filter(filter, @options))
+    end
+
+    test("Arrow::ChunkedArray") do
+      chunks = [
+        Arrow::BooleanArray.new([nil, true]),
+        Arrow::BooleanArray.new([true, false]),
+      ]
+      filter = Arrow::ChunkedArray.new(chunks)
+      filtered_array = Arrow::BooleanArray.new([nil, false, false])
+      assert_equal(filtered_array,
+                   @array.filter(filter, @options))
+    end
+  end
+
+  sub_test_case("#take") do
+    def setup
+      values = [1, 0 ,2]
+      @array = Arrow::Int16Array.new(values)
+    end
+
+    test("Arrow: boolean") do
+      indices = [1, 0, 2]
+      assert_equal(Arrow::Int16Array.new([0, 1, 2]),
+                   @array.take(indices))
+    end
+
+    test("Arrow::Array") do
+      indices = Arrow::Int16Array.new([1, 0, 2])
+      assert_equal(Arrow::Int16Array.new([0, 1, 2]),
+                   @array.take(indices))
+    end
+
+    test("Arrow::ChunkedArray") do
+      taken_chunks = [
+        Arrow::Int16Array.new([0, 1]),
+        Arrow::Int16Array.new([2])
+      ]
+      taken_chunked_array = Arrow::ChunkedArray.new(taken_chunks)
+      indices_chunks = [
+        Arrow::Int16Array.new([1, 0]),
+        Arrow::Int16Array.new([2])
+      ]
+      indices = Arrow::ChunkedArray.new(indices_chunks)
+      assert_equal(taken_chunked_array,
+                   @array.take(indices))
+    end
+  end
+
+  sub_test_case("#is_in") do
+    def setup
+      values = [1, 0, 1, 2]
+      @array = Arrow::Int16Array.new(values)
+    end
+
+    test("Arrow: Array") do
+      right = [2, 0]
+      assert_equal(Arrow::BooleanArray.new([false, true, false, true]),
+                   @array.is_in(right))
+    end
+
+    test("Arrow::Array") do
+      right = Arrow::Int16Array.new([2, 0])
+      assert_equal(Arrow::BooleanArray.new([false, true, false, true]),
+                   @array.is_in(right))
+    end
+
+    test("Arrow::ChunkedArray") do
+      chunks = [
+        Arrow::Int16Array.new([1, 0]),
+        Arrow::Int16Array.new([1, 0, 3])
+      ]
+      right = Arrow::ChunkedArray.new(chunks)
+      assert_equal(Arrow::BooleanArray.new([true, true, true, false]),
+                   @array.is_in(right))
+    end
   end
 end

data/test/test-buffer.rb

@@ -16,6 +16,17 @@
 # under the License.
 
 class BufferTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("GC") do
+      data = "Hello"
+      data_id = data.object_id
+      _buffer = Arrow::Buffer.new(data)
+      data = nil
+      GC.start
+      assert_equal("Hello", ObjectSpace._id2ref(data_id))
+    end
+  end
+
   sub_test_case("instance methods") do
     def setup
       @buffer = Arrow::Buffer.new("Hello")

data/test/test-chunked-array.rb

@@ -84,4 +84,100 @@ class ChunkedArrayTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case("#filter") do
+    def setup
+      arrays = [
+        Arrow::BooleanArray.new([false, true]),
+        Arrow::BooleanArray.new([false, true, false]),
+      ]
+      @chunked_array = Arrow::ChunkedArray.new(arrays)
+      @options = Arrow::FilterOptions.new
+      @options.null_selection_behavior = :emit_null
+    end
+
+    test("Array: boolean") do
+      filter = [nil, true, true, false, true]
+      chunks = [
+        Arrow::BooleanArray.new([nil, true]),
+        Arrow::BooleanArray.new([false, false]),
+      ]
+      filtered_chunked_array = Arrow::ChunkedArray.new(chunks)
+      assert_equal(filtered_chunked_array,
+                   @chunked_array.filter(filter, @options))
+    end
+
+    test("Arrow::BooleanArray") do
+      filter = Arrow::BooleanArray.new([nil, true, true, false, true])
+      chunks = [
+        Arrow::BooleanArray.new([nil, true]),
+        Arrow::BooleanArray.new([false, false]),
+      ]
+      filtered_chunked_array = Arrow::ChunkedArray.new(chunks)
+      assert_equal(filtered_chunked_array,
+                   @chunked_array.filter(filter, @options))
+    end
+
+    test("Arrow::ChunkedArray") do
+      chunks = [
+        Arrow::BooleanArray.new([nil, true]),
+        Arrow::BooleanArray.new([true, false, true]),
+      ]
+      filter = Arrow::ChunkedArray.new(chunks)
+      filtered_chunks = [
+        Arrow::BooleanArray.new([nil, true]),
+        Arrow::BooleanArray.new([false, false]),
+      ]
+      filtered_chunked_array = Arrow::ChunkedArray.new(filtered_chunks)
+      assert_equal(filtered_chunked_array,
+                   @chunked_array.filter(filter, @options))
+    end
+  end
+
+  sub_test_case("#take") do
+    def setup
+      chunks = [
+        Arrow::Int16Array.new([1, 0]),
+        Arrow::Int16Array.new([2]),
+      ]
+      @chunked_array = Arrow::ChunkedArray.new(chunks)
+    end
+
+    test("Arrow: boolean") do
+      chunks = [
+        Arrow::Int16Array.new([0, 1]),
+        Arrow::Int16Array.new([2])
+      ]
+      taken_chunked_array = Arrow::ChunkedArray.new(chunks)
+      indices = [1, 0, 2]
+      assert_equal(taken_chunked_array,
+                   @chunked_array.take(indices))
+    end
+
+    test("Arrow::Array") do
+      chunks = [
+        Arrow::Int16Array.new([0, 1]),
+        Arrow::Int16Array.new([2])
+      ]
+      taken_chunked_array = Arrow::ChunkedArray.new(chunks)
+      indices = Arrow::Int16Array.new([1, 0, 2])
+      assert_equal(taken_chunked_array,
+                   @chunked_array.take(indices))
+    end
+
+    test("Arrow::ChunkedArray") do
+      taken_chunks = [
+        Arrow::Int16Array.new([0, 1]),
+        Arrow::Int16Array.new([2])
+      ]
+      taken_chunked_array = Arrow::ChunkedArray.new(taken_chunks)
+      indices_chunks = [
+        Arrow::Int16Array.new([1, 0]),
+        Arrow::Int16Array.new([2])
+      ]
+      indices = Arrow::ChunkedArray.new(indices_chunks)
+      assert_equal(taken_chunked_array,
+                   @chunked_array.take(indices))
+    end
+  end
 end

data/test/test-csv-loader.rb

@@ -117,8 +117,8 @@ class CSVLoaderTest < Test::Unit::TestCase
   end
 
   sub_test_case("CSVReader") do
-    def load_csv(data, options)
-      Arrow::CSVLoader.load(data, options)
+    def load_csv(data, **options)
+      Arrow::CSVLoader.load(data, **options)
     end
 
     sub_test_case(":headers") do

data/test/test-data-type.rb

@@ -48,6 +48,17 @@ class DataTypeTest < Test::Unit::TestCase
       assert_equal(Arrow::FixedSizeBinaryDataType.new(10),
                    Arrow::DataType.resolve([:fixed_size_binary, 10]))
     end
+
+    test("abstract") do
+      message =
+        "abstract type: <:floating_point>: " +
+        "use one of not abstract type: [" +
+        "Arrow::DoubleDataType, " +
+        "Arrow::FloatDataType]"
+      assert_raise(ArgumentError.new(message)) do
+        Arrow::DataType.resolve(:floating_point)
+      end
+    end
   end
 
   sub_test_case("instance methods") do

data/test/test-dense-union-data-type.rb

@@ -28,12 +28,12 @@ class DenseUnionDataTypeTest < Test::Unit::TestCase
     end
 
     test("ordered arguments") do
-      assert_equal("union[dense]<visible: bool=2, count: int32=9>",
+      assert_equal("dense_union<visible: bool=2, count: int32=9>",
                    Arrow::DenseUnionDataType.new(@fields, [2, 9]).to_s)
     end
 
     test("description") do
-      assert_equal("union[dense]<visible: bool=2, count: int32=9>",
+      assert_equal("dense_union<visible: bool=2, count: int32=9>",
                    Arrow::DenseUnionDataType.new(fields: @fields,
                                                   type_codes: [2, 9]).to_s)
     end

data/test/test-dictionary-array.rb

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class DictionaryArrayTest < Test::Unit::TestCase
+  sub_test_case("instance methods") do
+    def setup
+      @values = ["a", "b", "c", "b", "a"]
+      @string_array = Arrow::StringArray.new(@values)
+      @array = @string_array.dictionary_encode
+    end
+
+    test("#[]") do
+      assert_equal(@values, @array.to_a)
+    end
+
+    test("#get_value") do
+      assert_equal([
+                     @values[0],
+                     @values[3],
+                   ],
+                   [
+                     @array.get_value(0),
+                     @array.get_value(3),
+                   ])
+    end
+  end
+end

data/test/test-feather.rb

@@ -18,17 +18,32 @@
 class FeatherTest < Test::Unit::TestCase
   include Helper::Fixture
 
-  def test_save_load
+  def setup
     columns = {
       "message" => Arrow::StringArray.new(["Start", "Crash", "Shutdown"]),
       "is_critical" => Arrow::BooleanArray.new([false, true, false]),
     }
-    table = Arrow::Table.new(columns)
+    @table = Arrow::Table.new(columns)
 
-    output = Tempfile.new(["red-arrow", ".feather"])
-    table.save(output.path)
-    output.close
+    @output = Tempfile.new(["red-arrow", ".feather"])
+    begin
+      yield(@output)
+    ensure
+      @output.close!
+    end
+  end
+
+  def test_default
+    @table.save(@output.path)
+    @output.close
+
+    assert_equal(@table, Arrow::Table.load(@output.path))
+  end
+
+  def test_compression
+    @table.save(@output.path, compression: :zstd)
+    @output.close
 
-    assert_equal(table, Arrow::Table.load(output.path))
+    assert_equal(@table, Arrow::Table.load(@output.path))
   end
 end

data/test/test-list-data-type.rb

@@ -23,7 +23,7 @@ class ListDataTypeTest < Test::Unit::TestCase
                    Arrow::ListDataType.new(field).to_s)
     end
 
-    test("Hash") do
+    test("name: String") do
       assert_equal("list<tag: string>",
                    Arrow::ListDataType.new(name: "tag", type: :string).to_s)
     end
@@ -39,5 +39,31 @@ class ListDataTypeTest < Test::Unit::TestCase
       assert_equal("list<tag: string>",
                    Arrow::ListDataType.new(field: field_description).to_s)
     end
+
+    test("Arrow::DataType") do
+      data_type = Arrow::BooleanDataType.new
+      assert_equal("list<item: bool>",
+                   Arrow::ListDataType.new(data_type).to_s)
+    end
+
+    test("String") do
+      assert_equal("list<item: bool>",
+                   Arrow::ListDataType.new("boolean").to_s)
+    end
+
+    test("Symbol") do
+      assert_equal("list<item: bool>",
+                   Arrow::ListDataType.new(:boolean).to_s)
+    end
+
+    test("[data type name, additional information]") do
+      assert_equal("list<item: time32[ms]>",
+                   Arrow::ListDataType.new([:time32, :milli]).to_s)
+    end
+
+    test("type: Symbol") do
+      assert_equal("list<item: bool>",
+                   Arrow::ListDataType.new(type: :boolean).to_s)
+    end
   end
 end