red-arrow 0.15.1 → 1.0.1

data/test/test-null-array.rb

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class NullArrayTest < Test::Unit::TestCase
+  test("#[]") do
+    array = Arrow::NullArray.new(1)
+    assert_nil(array[0])
+  end
+end

data/test/test-record-batch-iterator.rb

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class RecordBatchIteratorTest < Test::Unit::TestCase
+  def setup
+    @schema = Arrow::Schema.new(visible: :boolean,
+                                count: :uint32)
+    @record_batches = [
+      Arrow::RecordBatch.new(@schema,
+                             visible: [true],
+                             count: [1]),
+      Arrow::RecordBatch.new(@schema,
+                             visible: [false, nil],
+                             count: [nil, 3]),
+    ]
+    @iterator = Arrow::RecordBatchIterator.new(@record_batches)
+  end
+
+  def test_to_a
+    assert_equal(@record_batches,
+                 @iterator.to_a)
+  end
+end

data/test/test-record-batch.rb

@@ -22,6 +22,20 @@ class RecordBatchTest < Test::Unit::TestCase
                                   count: :uint32)
     end
 
+    test("[raw_table]") do
+      raw_table = {
+        visible: [true, nil, false],
+        count: [1, nil, 3],
+      }
+      record_batch = Arrow::RecordBatch.new(raw_table)
+      assert_equal([
+                     {"visible" => true,  "count" => 1},
+                     {"visible" => nil,   "count" => nil},
+                     {"visible" => false, "count" => 3},
+                   ],
+                   record_batch.each_record.collect(&:to_h))
+    end
+
     test("[Schema, records]") do
       records = [
         {visible: true, count: 1},

data/test/test-schema.rb

@@ -16,6 +16,8 @@
 # under the License.
 
 class SchemaTest < Test::Unit::TestCase
+  include Helper::Omittable
+
   def setup
     @count_field = Arrow::Field.new("count", :uint32)
     @visible_field = Arrow::Field.new("visible", :boolean)
@@ -114,5 +116,19 @@ class SchemaTest < Test::Unit::TestCase
         end
       end
     end
+
+    sub_test_case("#to_s") do
+      test("show_metadata") do
+        require_gi_bindings(3, 4, 2)
+
+        schema = @schema.with_metadata("key" => "value")
+        assert_equal(<<-SCHEMA.chomp, schema.to_s(show_metadata: true))
+count: uint32
+visible: bool
+-- metadata --
+key: value
+        SCHEMA
+      end
+    end
   end
 end

data/test/test-slicer.rb

@@ -46,10 +46,14 @@ class SlicerTest < Test::Unit::TestCase
       end
       assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
-0	    1	true   
-1	    8	true   
-2	   16	true   
-3	  256	true   
+0	     	       
+1	    1	true   
+2	     	       
+3	    8	true   
+4	   16	true   
+5	     	       
+6	     	       
+7	  256	true   
       TABLE
     end
 
@@ -66,7 +70,8 @@ class SlicerTest < Test::Unit::TestCase
 4	   16	true   
 5	   32	false  
 6	   64	       
-7	  256	true   
+7	     	       
+8	  256	true   
       TABLE
     end
   end
@@ -78,8 +83,12 @@ class SlicerTest < Test::Unit::TestCase
       end
       assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
-0	    2	false  
-1	   32	false  
+0	     	       
+1	    2	false  
+2	     	       
+3	   32	false  
+4	     	       
+5	     	       
       TABLE
     end
 
@@ -90,6 +99,7 @@ class SlicerTest < Test::Unit::TestCase
       assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
 0	    0	       
+1	     	       
       TABLE
     end
   end
@@ -142,10 +152,14 @@ class SlicerTest < Test::Unit::TestCase
       end
       assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
-0	    1	true   
-1	    8	true   
-2	   16	true   
-3	  256	true   
+0	     	       
+1	    1	true   
+2	     	       
+3	    8	true   
+4	   16	true   
+5	     	       
+6	     	       
+7	  256	true   
       TABLE
     end
   end
@@ -172,8 +186,12 @@ class SlicerTest < Test::Unit::TestCase
       end
       assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
-0	    2	false  
-1	   32	false  
+0	     	       
+1	    2	false  
+2	     	       
+3	   32	false  
+4	     	       
+5	     	       
       TABLE
     end
   end
@@ -200,8 +218,12 @@ class SlicerTest < Test::Unit::TestCase
       end
       assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
-0	    2	false  
-1	   32	false  
+0	     	       
+1	    2	false  
+2	     	       
+3	   32	false  
+4	     	       
+5	     	       
       TABLE
     end
   end
@@ -217,6 +239,7 @@ class SlicerTest < Test::Unit::TestCase
 2	    2	false  
 3	    4	       
 4	    8	true   
+5	     	       
     TABLE
   end
 
@@ -229,7 +252,8 @@ class SlicerTest < Test::Unit::TestCase
 0	   16	true   
 1	   32	false  
 2	   64	       
-3	  256	true   
+3	     	       
+4	  256	true   
     TABLE
   end
 
@@ -245,6 +269,7 @@ class SlicerTest < Test::Unit::TestCase
 3	    4	       
 4	    8	true   
 5	   16	true   
+6	     	       
     TABLE
   end
 
@@ -256,7 +281,8 @@ class SlicerTest < Test::Unit::TestCase
 	count	visible
 0	   32	false  
 1	   64	       
-2	  256	true   
+2	     	       
+3	  256	true   
     TABLE
   end
 
@@ -268,7 +294,8 @@ class SlicerTest < Test::Unit::TestCase
 	count	visible
 0	   32	false  
 1	   64	       
-2	  256	true   
+2	     	       
+3	  256	true   
     TABLE
   end
 
@@ -284,6 +311,7 @@ class SlicerTest < Test::Unit::TestCase
 3	    4	       
 4	    8	true   
 5	   16	true   
+6	     	       
     TABLE
   end
 
@@ -296,7 +324,8 @@ class SlicerTest < Test::Unit::TestCase
 0	   16	true   
 1	   32	false  
 2	   64	       
-3	  256	true   
+3	     	       
+4	  256	true   
     TABLE
   end
 
@@ -311,6 +340,7 @@ class SlicerTest < Test::Unit::TestCase
 2	    2	false  
 3	    4	       
 4	    8	true   
+5	     	       
     TABLE
   end
 
@@ -324,6 +354,7 @@ class SlicerTest < Test::Unit::TestCase
 1	    4	       
 2	   16	true   
 3	   64	       
+4	     	       
     TABLE
   end
 
@@ -337,7 +368,8 @@ class SlicerTest < Test::Unit::TestCase
 1	    2	false  
 2	    8	true   
 3	   32	false  
-4	  256	true   
+4	     	       
+5	  256	true   
     TABLE
   end
 
@@ -347,8 +379,12 @@ class SlicerTest < Test::Unit::TestCase
     end
     assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
-0	   16	true   
-1	  256	true   
+0	     	       
+1	     	       
+2	   16	true   
+3	     	       
+4	     	       
+5	  256	true   
     TABLE
   end
 
@@ -358,11 +394,15 @@ class SlicerTest < Test::Unit::TestCase
     end
     assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
-0	    1	true   
-1	    8	true   
-2	   16	true   
-3	   32	false  
-4	  256	true   
+0	     	       
+1	    1	true   
+2	     	       
+3	    8	true   
+4	   16	true   
+5	   32	false  
+6	     	       
+7	     	       
+8	  256	true   
     TABLE
   end
 
@@ -372,9 +412,13 @@ class SlicerTest < Test::Unit::TestCase
     end
     assert_equal(<<-TABLE, sliced_table.to_s)
 	count	visible
-0	    1	true   
-1	    8	true   
-2	   32	false  
+0	     	       
+1	    1	true   
+2	     	       
+3	    8	true   
+4	   32	false  
+5	     	       
+6	     	       
     TABLE
   end
 

data/test/test-sparse-union-data-type.rb

@@ -28,12 +28,12 @@ class SparseUnionDataTypeTest < Test::Unit::TestCase
     end
 
     test("ordered arguments") do
-      assert_equal("union[sparse]<visible: bool=2, count: int32=9>",
+      assert_equal("sparse_union<visible: bool=2, count: int32=9>",
                    Arrow::SparseUnionDataType.new(@fields, [2, 9]).to_s)
     end
 
     test("description") do
-      assert_equal("union[sparse]<visible: bool=2, count: int32=9>",
+      assert_equal("sparse_union<visible: bool=2, count: int32=9>",
                    Arrow::SparseUnionDataType.new(fields: @fields,
                                                   type_codes: [2, 9]).to_s)
     end

data/test/test-struct-array-builder.rb

@@ -38,10 +38,12 @@ class StructArrayBuilderTest < Test::Unit::TestCase
 
     test("Array") do
       @builder.append_value([true, 1])
+      @builder.append_value([])
+      @builder.append_value([false])
       array = @builder.finish
       assert_equal([
-                     [true],
-                     [1],
+                     [true, nil, false],
+                     [1, nil, nil],
                    ],
                    [
                      array.find_field(0).to_a,
@@ -66,10 +68,12 @@ class StructArrayBuilderTest < Test::Unit::TestCase
 
     test("Hash") do
       @builder.append_value(count: 1, visible: true)
+      @builder.append_value(visible: false)
+      @builder.append_value(count: 2)
       array = @builder.finish
       assert_equal([
-                     [true],
-                     [1],
+                     [true, false, nil],
+                     [1, nil, 2],
                    ],
                    [
                      array.find_field(0).to_a,

data/test/test-table.rb

@@ -54,11 +54,12 @@ class TableTest < Test::Unit::TestCase
       target_rows = Arrow::BooleanArray.new(target_rows_raw)
       assert_equal(<<-TABLE, @table.slice(target_rows).to_s)
 	count	visible
-0	    2	false  
-1	    4	       
-2	   16	true   
-3	   64	       
-4	  128	       
+0	     	       
+1	    2	false  
+2	    4	       
+3	   16	true   
+4	   64	       
+5	  128	       
       TABLE
     end
 
@@ -66,11 +67,12 @@ class TableTest < Test::Unit::TestCase
       target_rows_raw = [nil, true, true, false, true, false, true, true]
       assert_equal(<<-TABLE, @table.slice(target_rows_raw).to_s)
 	count	visible
-0	    2	false  
-1	    4	       
-2	   16	true   
-3	   64	       
-4	  128	       
+0	     	       
+1	    2	false  
+2	    4	       
+3	   16	true   
+4	   64	       
+5	  128	       
       TABLE
     end
 
@@ -436,12 +438,24 @@ class TableTest < Test::Unit::TestCase
         assert_equal(@table, Arrow::Table.load(output))
       end
 
+      def test_arrow_file
+        output = create_output(".arrow")
+        @table.save(output, format: :arrow_file)
+        assert_equal(@table, Arrow::Table.load(output, format: :arrow_file))
+      end
+
       def test_batch
         output = create_output(".arrow")
         @table.save(output, format: :batch)
         assert_equal(@table, Arrow::Table.load(output, format: :batch))
       end
 
+      def test_arrow_streaming
+        output = create_output(".arrow")
+        @table.save(output, format: :arrow_streaming)
+        assert_equal(@table, Arrow::Table.load(output, format: :arrow_streaming))
+      end
+
       def test_stream
         output = create_output(".arrow")
         @table.save(output, format: :stream)
@@ -468,6 +482,15 @@ class TableTest < Test::Unit::TestCase
                                        compression: :gzip,
                                        schema: @table.schema))
       end
+
+      def test_tsv
+        output = create_output(".tsv")
+        @table.save(output, format: :tsv)
+        assert_equal(@table,
+                     Arrow::Table.load(output,
+                                       format: :tsv,
+                                       schema: @table.schema))
+      end
     end
 
     sub_test_case("path") do
@@ -498,18 +521,27 @@ class TableTest < Test::Unit::TestCase
                                            compression: :gzip,
                                            schema: @table.schema))
           end
+
+          test("tsv") do
+            output = create_output(".tsv")
+            @table.save(output)
+            assert_equal(@table,
+                         Arrow::Table.load(output,
+                                           format: :tsv,
+                                           schema: @table.schema))
+          end
         end
 
         sub_test_case("load: auto detect") do
-          test("batch") do
+          test("arrow: file") do
             output = create_output(".arrow")
-            @table.save(output, format: :batch)
+            @table.save(output, format: :arrow_file)
             assert_equal(@table, Arrow::Table.load(output))
           end
 
-          test("stream") do
+          test("arrow: streaming") do
             output = create_output(".arrow")
-            @table.save(output, format: :stream)
+            @table.save(output, format: :arrow_streaming)
             assert_equal(@table, Arrow::Table.load(output))
           end
 
@@ -539,6 +571,24 @@ chris,-1
 	name	score
 0	alice	   10
 1	bob 	   29
+2	chris	   -1
+            TABLE
+          end
+
+          test("tsv") do
+            file = Tempfile.new(["red-arrow", ".tsv"])
+            file.puts(<<-TSV)
+name\tscore
+alice\t10
+bob\t29
+chris\t-1
+            TSV
+            file.close
+            table = Arrow::Table.load(file.path)
+            assert_equal(<<-TABLE, table.to_s)
+	name	score
+0	alice	   10
+1	bob 	   29
 2	chris	   -1
             TABLE
           end
@@ -646,4 +696,93 @@ visible: false
       end
     end
   end
+
+  sub_test_case("#filter") do
+    def setup
+      super
+      @options = Arrow::FilterOptions.new
+      @options.null_selection_behavior = :emit_null
+    end
+
+    test("Array: boolean") do
+      filter = [nil, true, true, false, true, false, true, true]
+      assert_equal(<<-TABLE, @table.filter(filter, @options).to_s)
+	count	visible
+0	     	       
+1	    2	false  
+2	    4	       
+3	   16	true   
+4	   64	       
+5	  128	       
+      TABLE
+    end
+
+    test("Arrow::BooleanArray") do
+      array = [nil, true, true, false, true, false, true, true]
+      filter = Arrow::BooleanArray.new(array)
+      assert_equal(<<-TABLE, @table.filter(filter, @options).to_s)
+	count	visible
+0	     	       
+1	    2	false  
+2	    4	       
+3	   16	true   
+4	   64	       
+5	  128	       
+      TABLE
+    end
+
+    test("Arrow::ChunkedArray") do
+      filter_chunks = [
+        Arrow::BooleanArray.new([nil, true, true]),
+        Arrow::BooleanArray.new([false, true, false]),
+        Arrow::BooleanArray.new([true, true]),
+      ]
+      filter = Arrow::ChunkedArray.new(filter_chunks)
+      assert_equal(<<-TABLE, @table.filter(filter, @options).to_s)
+	count	visible
+0	     	       
+1	    2	false  
+2	    4	       
+3	   16	true   
+4	   64	       
+5	  128	       
+      TABLE
+    end
+  end
+
+  sub_test_case("#take") do
+    test("Arrow: boolean") do
+      indices = [1, 0, 2]
+      assert_equal(<<-TABLE, @table.take(indices).to_s)
+	count	visible
+0	    2	false  
+1	    1	true   
+2	    4	       
+      TABLE
+    end
+
+    test("Arrow::Array") do
+      indices = Arrow::Int16Array.new([1, 0, 2])
+      assert_equal(<<-TABLE, @table.take(indices).to_s)
+	count	visible
+0	    2	false  
+1	    1	true   
+2	    4	       
+      TABLE
+    end
+
+    test("Arrow::ChunkedArray") do
+      chunks = [
+        Arrow::Int16Array.new([1, 0]),
+        Arrow::Int16Array.new([2])
+      ]
+      indices = Arrow::ChunkedArray.new(chunks)
+      assert_equal(<<-TABLE, @table.take(indices).to_s)
+	count	visible
+0	    2	false  
+1	    1	true   
+2	    4	       
+      TABLE
+    end
+  end
 end