red-arrow 0.8.1 → 0.8.2

data/lib/arrow/version.rb

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 module Arrow
-  VERSION = "0.8.1"
+  VERSION = "0.8.2"
 end

data/test/fixture/null-with-double-quote.csv

@@ -0,0 +1,4 @@
+name,score
+alice,10
+bob,""
+chris,-1

data/test/fixture/null-without-double-quote.csv

@@ -0,0 +1,4 @@
+name,score
+alice,10
+bob,
+chris,-1

data/test/run-test.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 #
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Copyright 2017-2018 Kouhei Sutou <kou@clear-code.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ENV["TZ"] = "Asia/Tokyo"
+
 $VERBOSE = true
 
 require "pathname"

data/test/test-chunked-array.rb

@@ -14,12 +14,49 @@
 
 class ChunkedArrayTest < Test::Unit::TestCase
   test("#each") do
-    arrayes = [
+    arrays = [
       Arrow::BooleanArray.new([true, false]),
       Arrow::BooleanArray.new([nil, true]),
     ]
-    chunked_array = Arrow::ChunkedArray.new(arrayes)
+    chunked_array = Arrow::ChunkedArray.new(arrays)
     assert_equal([true, false, nil, true],
                  chunked_array.to_a)
   end
+
+  sub_test_case("#pack") do
+    test("basic array") do
+      arrays = [
+        Arrow::BooleanArray.new([true, false]),
+        Arrow::BooleanArray.new([nil, true]),
+      ]
+      chunked_array = Arrow::ChunkedArray.new(arrays)
+      packed_chunked_array = chunked_array.pack
+      assert_equal([
+                     Arrow::BooleanArray,
+                     [true, false, nil, true],
+                   ],
+                   [
+                     packed_chunked_array.class,
+                     packed_chunked_array.to_a,
+                   ])
+    end
+
+    test("TimestampArray") do
+      type = Arrow::TimestampDataType.new(:nano)
+      arrays = [
+        Arrow::TimestampArrayBuilder.new(type).build([Time.at(0)]),
+        Arrow::TimestampArrayBuilder.new(type).build([Time.at(1)]),
+      ]
+      chunked_array = Arrow::ChunkedArray.new(arrays)
+      packed_chunked_array = chunked_array.pack
+      assert_equal([
+                     Arrow::TimestampArray,
+                     [Time.at(0), Time.at(1)],
+                   ],
+                   [
+                     packed_chunked_array.class,
+                     packed_chunked_array.to_a,
+                   ])
+    end
+  end
 end

data/test/test-column.rb

@@ -24,4 +24,17 @@ class ColumnTest < Test::Unit::TestCase
     assert_equal([true, false, nil, true],
                  column.to_a)
   end
+
+  test("#pack") do
+    arrays = [
+      Arrow::BooleanArray.new([true, false]),
+      Arrow::BooleanArray.new([nil, true]),
+    ]
+    chunked_array = Arrow::ChunkedArray.new(arrays)
+    column = Arrow::Column.new(Arrow::Field.new("visible", :boolean),
+                               chunked_array)
+    packed_column = column.pack
+    assert_equal([1, [true, false, nil, true]],
+                 [packed_column.data.n_chunks, packed_column.to_a])
+  end
 end

data/test/test-csv-loader.rb

@@ -75,5 +75,25 @@ class CSVLoaderTest < Test::Unit::TestCase
 2	chris	-1
       TABLE
     end
+
+    test("null: with double quote") do
+      path = fixture_path("null-with-double-quote.csv").to_s
+      assert_equal(<<-TABLE, Arrow::CSVLoader.load(path).to_s)
+	name	score
+0	alice	   10
+1	bob 	     
+2	chris	   -1
+      TABLE
+    end
+
+    test("null: without double quote") do
+      path = fixture_path("null-without-double-quote.csv").to_s
+      assert_equal(<<-TABLE, Arrow::CSVLoader.load(path).to_s)
+	name	score
+0	alice	   10
+1	bob 	     
+2	chris	   -1
+      TABLE
+    end
   end
 end

data/test/test-group.rb

@@ -0,0 +1,111 @@
+# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class GroupTest < Test::Unit::TestCase
+  include Helper::Fixture
+
+  def setup
+    raw_table = {
+      :group_key1 => Arrow::UInt8Array.new([1, 1, 2, 3, 3, 3]),
+      :group_key2 => Arrow::UInt8Array.new([1, 1, 1, 1, 2, 2]),
+      :int => Arrow::Int32Array.new([-1, -2, nil, -4, -5, -6]),
+      :uint => Arrow::UInt32Array.new([1, nil, 3, 4, 5, 6]),
+      :float => Arrow::FloatArray.new([nil, 2.2, 3.3, 4.4, 5.5, 6.6]),
+      :string => Arrow::StringArray.new(["a", "b", "c", nil, "e", "f"]),
+    }
+    @table = Arrow::Table.new(raw_table)
+  end
+
+  sub_test_case("key") do
+    test("Time") do
+      time_values = [
+        Time.parse("2018-01-29"),
+        Time.parse("2018-01-30"),
+      ]
+      raw_table = {
+        :time => Arrow::ArrayBuilder.build(time_values),
+        :int => Arrow::Int32Array.new([-1, -2]),
+      }
+      table = Arrow::Table.new(raw_table)
+      assert_equal(<<-TABLE, table.group(:time).count.to_s)
+	                     time	int
+0	2018-01-29T00:00:00+09:00	  1
+1	2018-01-30T00:00:00+09:00	  1
+      TABLE
+    end
+  end
+
+  sub_test_case("#count") do
+    test("single") do
+      assert_equal(<<-TABLE, @table.group(:group_key1).count.to_s)
+	group_key1	group_key2	int	uint	float	string
+0	         1	         2	  2	   1	    1	     2
+1	         2	         1	  0	   1	    1	     1
+2	         3	         3	  3	   3	    3	     2
+      TABLE
+    end
+
+    test("multiple") do
+      assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).count.to_s)
+	group_key1	group_key2	int	uint	float	string
+0	         1	         1	  2	   1	    1	     2
+1	         2	         1	  0	   1	    1	     1
+2	         3	         1	  1	   1	    1	     0
+3	         3	         2	  2	   2	    2	     2
+      TABLE
+    end
+  end
+
+  sub_test_case("#sum") do
+    test("single") do
+      assert_equal(<<-TABLE, @table.group(:group_key1).sum.to_s)
+	group_key1	group_key2	int	uint	     float
+0	         1	         2	 -3	   1	  2.200000
+1	         2	         1	  0	   3	  3.300000
+2	         3	         5	-15	  15	 16.500000
+      TABLE
+    end
+
+    test("multiple") do
+      assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).sum.to_s)
+	group_key1	group_key2	int	uint	     float
+0	         1	         1	 -3	   1	  2.200000
+1	         2	         1	  0	   3	  3.300000
+2	         3	         1	 -4	   4	  4.400000
+3	         3	         2	-11	  11	 12.100000
+      TABLE
+    end
+  end
+
+  sub_test_case("#average") do
+    test("single") do
+      assert_equal(<<-TABLE, @table.group(:group_key1).average.to_s)
+	group_key1	group_key2	       int	      uint	     float
+0	         1	  1.000000	 -1.500000	  1.000000	  2.200000
+1	         2	  1.000000	  0.000000	  3.000000	  3.300000
+2	         3	  1.666667	 -5.000000	  5.000000	  5.500000
+      TABLE
+    end
+
+    test("multiple") do
+      assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).average.to_s)
+	group_key1	group_key2	       int	      uint	     float
+0	         1	         1	 -1.500000	  1.000000	  2.200000
+1	         2	         1	  0.000000	  3.000000	  3.300000
+2	         3	         1	 -4.000000	  4.000000	  4.400000
+3	         3	         2	 -5.500000	  5.500000	  6.050000
+      TABLE
+    end
+  end
+end

data/test/test-slicer.rb

@@ -106,6 +106,21 @@ class SlicerTest < Test::Unit::TestCase
     TABLE
   end
 
+  test("column.valid?") do
+    sliced_table = @table.slice do |slicer|
+      slicer.visible.valid?
+    end
+    assert_equal(<<-TABLE, sliced_table.to_s)
+	count	visible
+0	    1	true   
+1	    2	false  
+2	    8	true   
+3	   16	true   
+4	   32	false  
+5	  256	true   
+    TABLE
+  end
+
   sub_test_case("column ==") do
     test("nil") do
       sliced_table = @table.slice do |slicer|
@@ -298,6 +313,33 @@ class SlicerTest < Test::Unit::TestCase
     TABLE
   end
 
+  test("column.in") do
+    sliced_table = @table.slice do |slicer|
+      slicer.count.in?([1, 4, 16, 64])
+    end
+    assert_equal(<<-TABLE, sliced_table.to_s)
+	count	visible
+0	    1	true   
+1	    4	       
+2	   16	true   
+3	   64	       
+    TABLE
+  end
+
+  test("!column.in") do
+    sliced_table = @table.slice do |slicer|
+      !slicer.count.in?([1, 4, 16, 64])
+    end
+    assert_equal(<<-TABLE, sliced_table.to_s)
+	count	visible
+0	    0	       
+1	    2	false  
+2	    8	true   
+3	   32	false  
+4	  256	true   
+    TABLE
+  end
+
   test("condition & condition") do
     sliced_table = @table.slice do |slicer|
       slicer.visible & (slicer.count >= 16)

data/test/test-table.rb

@@ -58,14 +58,21 @@ class TableTest < Test::Unit::TestCase
       TABLE
     end
 
-    test("Integer") do
+    test("Integer: positive") do
       assert_equal(<<-TABLE, @table.slice(2).to_s)
 	count	visible
 0	    4	       
       TABLE
     end
 
-    test("Range: include end") do
+    test("Integer: negative") do
+      assert_equal(<<-TABLE, @table.slice(-1).to_s)
+	count	visible
+0	  128	       
+      TABLE
+    end
+
+    test("Range: positive: include end") do
       assert_equal(<<-TABLE, @table.slice(2..4).to_s)
 	count	visible
 0	    4	       
@@ -74,7 +81,7 @@ class TableTest < Test::Unit::TestCase
       TABLE
     end
 
-    test("Range: exclude end") do
+    test("Range: positive: exclude end") do
       assert_equal(<<-TABLE, @table.slice(2...4).to_s)
 	count	visible
 0	    4	       
@@ -82,7 +89,24 @@ class TableTest < Test::Unit::TestCase
       TABLE
     end
 
-    test("[from, to]") do
+    test("Range: negative: include end") do
+      assert_equal(<<-TABLE, @table.slice(-4..-2).to_s)
+	count	visible
+0	   16	true   
+1	   32	false  
+2	   64	       
+      TABLE
+    end
+
+    test("Range: negative: exclude end") do
+      assert_equal(<<-TABLE, @table.slice(-4...-2).to_s)
+	count	visible
+0	   16	true   
+1	   32	false  
+      TABLE
+    end
+
+    test("[from, to]: positive") do
       assert_equal(<<-TABLE, @table.slice([0, 2]).to_s)
 	count	visible
 0	    1	true   
@@ -90,6 +114,14 @@ class TableTest < Test::Unit::TestCase
       TABLE
     end
 
+    test("[from, to]: negative") do
+      assert_equal(<<-TABLE, @table.slice([-4, 2]).to_s)
+	count	visible
+0	   16	true   
+1	   32	false  
+      TABLE
+    end
+
     test("Integer, Range, ...") do
       assert_equal(<<-TABLE, @table.slice(0, 4...7).to_s)
 	count	visible
@@ -109,12 +141,6 @@ class TableTest < Test::Unit::TestCase
     test("[Symbol]") do
       assert_equal(@visible_column, @table[:visible])
     end
-
-    test("[String, Symbol]") do
-      assert_equal(Arrow::Table.new(@table.schema,
-                                    [@visible_column, @count_column]).to_s,
-                   @table["visible", :count].to_s)
-    end
   end
 
   sub_test_case("#merge") do
@@ -391,4 +417,20 @@ class TableTest < Test::Unit::TestCase
       end
     end
   end
+
+  test("#pack") do
+    packed_table = @table.pack
+    column_n_chunks = packed_table.columns.collect {|c| c.data.n_chunks}
+    assert_equal([[1, 1], <<-TABLE], [column_n_chunks, packed_table.to_s])
+	count	visible
+0	    1	true   
+1	    2	false  
+2	    4	       
+3	    8	true   
+4	   16	true   
+5	   32	false  
+6	   64	       
+7	  128	       
+    TABLE
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-05 00:00:00.000000000 Z
+date: 2018-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gobject-introspection
@@ -150,11 +150,13 @@ files:
 - lib/arrow/compatibility.rb
 - lib/arrow/csv-loader.rb
 - lib/arrow/csv-reader.rb
+- lib/arrow/data-type.rb
 - lib/arrow/date32-array-builder.rb
 - lib/arrow/date32-array.rb
 - lib/arrow/date64-array-builder.rb
 - lib/arrow/date64-array.rb
 - lib/arrow/field.rb
+- lib/arrow/group.rb
 - lib/arrow/loader.rb
 - lib/arrow/record-batch-file-reader.rb
 - lib/arrow/record-batch-stream-reader.rb
@@ -163,14 +165,18 @@ files:
 - lib/arrow/record.rb
 - lib/arrow/slicer.rb
 - lib/arrow/table-formatter.rb
+- lib/arrow/table-list-formatter.rb
 - lib/arrow/table-loader.rb
 - lib/arrow/table-saver.rb
+- lib/arrow/table-table-formatter.rb
 - lib/arrow/table.rb
 - lib/arrow/tensor.rb
 - lib/arrow/timestamp-array-builder.rb
 - lib/arrow/timestamp-array.rb
 - lib/arrow/version.rb
 - red-arrow.gemspec
+- test/fixture/null-with-double-quote.csv
+- test/fixture/null-without-double-quote.csv
 - test/fixture/with-header.csv
 - test/fixture/without-header.csv
 - test/helper.rb
@@ -184,6 +190,7 @@ files:
 - test/test-csv-reader.rb
 - test/test-date32-array.rb
 - test/test-date64-array.rb
+- test/test-group.rb
 - test/test-record-batch-file-reader.rb
 - test/test-record-batch.rb
 - test/test-slicer.rb
@@ -217,7 +224,9 @@ summary: Red Arrow is a Ruby bindings of Apache Arrow. Red Arrow is based on GOb
 test_files:
 - test/test-record-batch-file-reader.rb
 - test/fixture/with-header.csv
+- test/fixture/null-without-double-quote.csv
 - test/fixture/without-header.csv
+- test/fixture/null-with-double-quote.csv
 - test/test-csv-reader.rb
 - test/test-column.rb
 - test/test-array-builder.rb
@@ -230,6 +239,7 @@ test_files:
 - test/test-table.rb
 - test/test-csv-loader.rb
 - test/test-array.rb
+- test/test-group.rb
 - test/helper/fixture.rb
 - test/test-date64-array.rb
 - test/test-slicer.rb