red-arrow 10.0.1 → 12.0.0

data/test/test-group.rb

@@ -42,9 +42,9 @@ class GroupTest < Test::Unit::TestCase
       }
       table = Arrow::Table.new(raw_table)
       assert_equal(<<-TABLE, table.group(:time).count.to_s)
-	count(int)	                     time
-0	         1	#{time_values[0].iso8601}
-1	         1	#{time_values[1].iso8601}
+	                     time	count(int)
+0	#{time_values[0].iso8601}	         1
+1	#{time_values[1].iso8601}	         1
       TABLE
     end
   end
@@ -52,31 +52,31 @@ class GroupTest < Test::Unit::TestCase
   sub_test_case("#count") do
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).count.to_s)
-	count(group_key2)	count(int)	count(uint)	count(float)	count(string)	group_key1
-0	                2	         2	          1	           1	            2	         1
-1	                1	         0	          1	           1	            1	         2
-2	                3	         3	          3	           3	            2	         3
+	group_key1	count(group_key2)	count(int)	count(uint)	count(float)	count(string)
+0	         1	                2	         2	          1	           1	            2
+1	         2	                1	         0	          1	           1	            1
+2	         3	                3	         3	          3	           3	            2
       TABLE
     end
 
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).count.to_s)
-	count(int)	count(uint)	count(float)	count(string)	group_key1	group_key2
-0	         2	          1	           1	            2	         1	         1
-1	         0	          1	           1	            1	         2	         1
-2	         1	          1	           1	            0	         3	         1
-3	         2	          2	           2	            2	         3	         2
+	group_key1	group_key2	count(int)	count(uint)	count(float)	count(string)
+0	         1	         1	         2	          1	           1	            2
+1	         2	         1	         0	          1	           1	            1
+2	         3	         1	         1	          1	           1	            0
+3	         3	         2	         2	          2	           2	            2
       TABLE
     end
 
     test("column") do
       group = @table.group(:group_key1, :group_key2)
       assert_equal(<<-TABLE, group.count(:int, :uint).to_s)
-	count(int)	count(uint)	group_key1	group_key2
-0	         2	          1	         1	         1
-1	         0	          1	         2	         1
-2	         1	          1	         3	         1
-3	         2	          2	         3	         2
+	group_key1	group_key2	count(int)	count(uint)
+0	         1	         1	         2	          1
+1	         2	         1	         0	          1
+2	         3	         1	         1	          1
+3	         3	         2	         2	          2
       TABLE
     end
   end
@@ -84,20 +84,20 @@ class GroupTest < Test::Unit::TestCase
   sub_test_case("#sum") do
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).sum.to_s)
-	sum(group_key2)	sum(int)	sum(uint)	sum(float)	group_key1
-0	              2	      -3	        1	  2.200000	         1
-1	              1	  (null)	        3	  3.300000	         2
-2	              5	     -15	       15	 16.500000	         3
+	group_key1	sum(group_key2)	sum(int)	sum(uint)	sum(float)
+0	         1	              2	      -3	        1	  2.200000
+1	         2	              1	  (null)	        3	  3.300000
+2	         3	              5	     -15	       15	 16.500000
       TABLE
     end
 
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).sum.to_s)
-	sum(int)	sum(uint)	sum(float)	group_key1	group_key2
-0	      -3	        1	  2.200000	         1	         1
-1	  (null)	        3	  3.300000	         2	         1
-2	      -4	        4	  4.400000	         3	         1
-3	     -11	       11	 12.100000	         3	         2
+	group_key1	group_key2	sum(int)	sum(uint)	sum(float)
+0	         1	         1	      -3	        1	  2.200000
+1	         2	         1	  (null)	        3	  3.300000
+2	         3	         1	      -4	        4	  4.400000
+3	         3	         2	     -11	       11	 12.100000
       TABLE
     end
   end
@@ -105,20 +105,20 @@ class GroupTest < Test::Unit::TestCase
   sub_test_case("#mean") do
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).mean.to_s)
-	mean(group_key2)	 mean(int)	mean(uint)	mean(float)	group_key1
-0	        1.000000	 -1.500000	  1.000000	   2.200000	         1
-1	        1.000000	    (null)	  3.000000	   3.300000	         2
-2	        1.666667	 -5.000000	  5.000000	   5.500000	         3
+	group_key1	mean(group_key2)	 mean(int)	mean(uint)	mean(float)
+0	         1	        1.000000	 -1.500000	  1.000000	   2.200000
+1	         2	        1.000000	    (null)	  3.000000	   3.300000
+2	         3	        1.666667	 -5.000000	  5.000000	   5.500000
       TABLE
     end
 
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).mean.to_s)
-	 mean(int)	mean(uint)	mean(float)	group_key1	group_key2
-0	 -1.500000	  1.000000	   2.200000	         1	         1
-1	    (null)	  3.000000	   3.300000	         2	         1
-2	 -4.000000	  4.000000	   4.400000	         3	         1
-3	 -5.500000	  5.500000	   6.050000	         3	         2
+	group_key1	group_key2	 mean(int)	mean(uint)	mean(float)
+0	         1	         1	 -1.500000	  1.000000	   2.200000
+1	         2	         1	    (null)	  3.000000	   3.300000
+2	         3	         1	 -4.000000	  4.000000	   4.400000
+3	         3	         2	 -5.500000	  5.500000	   6.050000
       TABLE
     end
   end
@@ -126,20 +126,20 @@ class GroupTest < Test::Unit::TestCase
   sub_test_case("#min") do
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).min.to_s)
-	min(group_key2)	min(int)	min(uint)	min(float)	group_key1
-0	              1	      -2	        1	  2.200000	         1
-1	              1	  (null)	        3	  3.300000	         2
-2	              1	      -6	        4	  4.400000	         3
+	group_key1	min(group_key2)	min(int)	min(uint)	min(float)
+0	         1	              1	      -2	        1	  2.200000
+1	         2	              1	  (null)	        3	  3.300000
+2	         3	              1	      -6	        4	  4.400000
       TABLE
     end
 
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).min.to_s)
-	min(int)	min(uint)	min(float)	group_key1	group_key2
-0	      -2	        1	  2.200000	         1	         1
-1	  (null)	        3	  3.300000	         2	         1
-2	      -4	        4	  4.400000	         3	         1
-3	      -6	        5	  5.500000	         3	         2
+	group_key1	group_key2	min(int)	min(uint)	min(float)
+0	         1	         1	      -2	        1	  2.200000
+1	         2	         1	  (null)	        3	  3.300000
+2	         3	         1	      -4	        4	  4.400000
+3	         3	         2	      -6	        5	  5.500000
       TABLE
     end
   end
@@ -147,20 +147,20 @@ class GroupTest < Test::Unit::TestCase
   sub_test_case("#max") do
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).max.to_s)
-	max(group_key2)	max(int)	max(uint)	max(float)	group_key1
-0	              1	      -1	        1	  2.200000	         1
-1	              1	  (null)	        3	  3.300000	         2
-2	              2	      -4	        6	  6.600000	         3
+	group_key1	max(group_key2)	max(int)	max(uint)	max(float)
+0	         1	              1	      -1	        1	  2.200000
+1	         2	              1	  (null)	        3	  3.300000
+2	         3	              2	      -4	        6	  6.600000
       TABLE
     end
 
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).max.to_s)
-	max(int)	max(uint)	max(float)	group_key1	group_key2
-0	      -1	        1	  2.200000	         1	         1
-1	  (null)	        3	  3.300000	         2	         1
-2	      -4	        4	  4.400000	         3	         1
-3	      -5	        6	  6.600000	         3	         2
+	group_key1	group_key2	max(int)	max(uint)	max(float)
+0	         1	         1	      -1	        1	  2.200000
+1	         2	         1	  (null)	        3	  3.300000
+2	         3	         1	      -4	        4	  4.400000
+3	         3	         2	      -5	        6	  6.600000
       TABLE
     end
   end
@@ -169,11 +169,11 @@ class GroupTest < Test::Unit::TestCase
     test("function()") do
       group = @table.group(:group_key1, :group_key2)
       assert_equal(<<-TABLE, group.aggregate("count(int)", "sum(uint)").to_s)
-	count(int)	sum(uint)	group_key1	group_key2
-0	         2	        1	         1	         1
-1	         0	        3	         2	         1
-2	         1	        4	         3	         1
-3	         2	       11	         3	         2
+	group_key1	group_key2	count(int)	sum(uint)
+0	         1	         1	         2	        1
+1	         2	         1	         0	        3
+2	         3	         1	         1	        4
+3	         3	         2	         2	       11
       TABLE
     end
   end

data/test/test-half-float-array.rb

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class HalfFloatArrayTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("Float") do
+      array = Arrow::HalfFloatArray.new([1.5])
+      assert_equal([1.5], array.to_a)
+    end
+
+    test("Integer") do
+      one_half = Arrow::HalfFloat.new(1.5)
+      array = Arrow::HalfFloatArray.new([one_half.to_uint16])
+      assert_equal([one_half.to_f], array.to_a)
+    end
+
+    test("HalfFloat") do
+      one_half = Arrow::HalfFloat.new(1.5)
+      array = Arrow::HalfFloatArray.new([one_half])
+      assert_equal([one_half.to_f], array.to_a)
+    end
+  end
+
+  test("#[]") do
+    one_half = Arrow::HalfFloat.new(1.5)
+    array = Arrow::HalfFloatArray.new([one_half.to_uint16])
+    assert_equal(one_half.to_f, array[0])
+  end
+end

data/test/test-half-float.rb

@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class HalfFloatTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("Array") do
+      positive_infinity = Arrow::HalfFloat.new(0b1, 0b11111, 0b0000000000)
+      assert_equal([0b1, 0b11111, 0b0000000000],
+                   [
+                     positive_infinity.sign,
+                     positive_infinity.exponent,
+                     positive_infinity.fraction,
+                   ])
+    end
+
+    test("Integer - 0") do
+      zero = Arrow::HalfFloat.new(0)
+      assert_equal([0b0, 0b00000, 0b0000000000],
+                   [
+                     zero.sign,
+                     zero.exponent,
+                     zero.fraction,
+                   ])
+    end
+
+    test("Integer - +infinity") do
+      positive_infinity = Arrow::HalfFloat.new(0x7c00)
+      assert_equal([0b0, 0b11111, 0b0000000000],
+                   [
+                     positive_infinity.sign,
+                     positive_infinity.exponent,
+                     positive_infinity.fraction,
+                   ])
+    end
+
+    test("Integer - -infinity") do
+      negative_infinity = Arrow::HalfFloat.new(0xfc00)
+      assert_equal([0b1, 0b11111, 0b0000000000],
+                   [
+                     negative_infinity.sign,
+                     negative_infinity.exponent,
+                     negative_infinity.fraction,
+                   ])
+    end
+
+    test("Integer - 1/3") do
+      one_thirds = Arrow::HalfFloat.new(0x3555)
+      assert_equal([0b0, 0b01101, 0b0101010101],
+                   [
+                     one_thirds.sign,
+                     one_thirds.exponent,
+                     one_thirds.fraction,
+                   ])
+    end
+
+    test("Float - 0") do
+      zero = Arrow::HalfFloat.new(0.0)
+      assert_equal([0b0, 0b00000, 0b0000000000],
+                   [
+                     zero.sign,
+                     zero.exponent,
+                     zero.fraction,
+                   ])
+    end
+
+    test("Float - too large") do
+      positive_infinity = Arrow::HalfFloat.new(65504.1)
+      assert_equal([0b0, 0b11111, 0b0000000000],
+                   [
+                     positive_infinity.sign,
+                     positive_infinity.exponent,
+                     positive_infinity.fraction,
+                   ])
+    end
+
+    test("Float - +infinity") do
+      positive_infinity = Arrow::HalfFloat.new(Float::INFINITY)
+      assert_equal([0b0, 0b11111, 0b0000000000],
+                   [
+                     positive_infinity.sign,
+                     positive_infinity.exponent,
+                     positive_infinity.fraction,
+                   ])
+    end
+
+    test("Float - too small") do
+      negative_infinity = Arrow::HalfFloat.new(-65504.1)
+      assert_equal([0b1, 0b11111, 0b0000000000],
+                   [
+                     negative_infinity.sign,
+                     negative_infinity.exponent,
+                     negative_infinity.fraction,
+                   ])
+    end
+
+    test("Float - -infinity") do
+      negative_infinity = Arrow::HalfFloat.new(-Float::INFINITY)
+      assert_equal([0b1, 0b11111, 0b0000000000],
+                   [
+                     negative_infinity.sign,
+                     negative_infinity.exponent,
+                     negative_infinity.fraction,
+                   ])
+    end
+
+    test("Float - 1/3") do
+      one_thirds = Arrow::HalfFloat.new((2 ** -2) * (1 + 341 / 1024.0))
+      assert_equal([0b0, 0b01101, 0b0101010101],
+                   [
+                     one_thirds.sign,
+                     one_thirds.exponent,
+                     one_thirds.fraction,
+                   ])
+    end
+  end
+end

data/test/test-record-batch-file-reader.rb

@@ -112,4 +112,25 @@ class RecordBatchFileReaderTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case("#each") do
+    test("without block") do
+      buffer = Arrow::ResizableBuffer.new(1024)
+      Arrow::Table.new(number: [1, 2, 3]).save(buffer)
+      Arrow::BufferInputStream.open(buffer) do |input|
+        reader = Arrow::RecordBatchFileReader.new(input)
+        each = reader.each
+        assert_equal({
+                       size: 1,
+                       to_a: [
+                         Arrow::RecordBatch.new(number: [1, 2, 3]),
+                       ],
+                     },
+                     {
+                       size: each.size,
+                       to_a: each.to_a,
+                     })
+      end
+    end
+  end
 end

data/test/test-record-batch-stream-reader.rb

@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class RecordBatchStreamReaderTest < Test::Unit::TestCase
+  test("write/read") do
+    fields = [
+      Arrow::Field.new("uint8",  :uint8),
+      Arrow::Field.new("uint16", :uint16),
+      Arrow::Field.new("uint32", :uint32),
+      Arrow::Field.new("uint64", :uint64),
+      Arrow::Field.new("int8",   :int8),
+      Arrow::Field.new("int16",  :int16),
+      Arrow::Field.new("int32",  :int32),
+      Arrow::Field.new("int64",  :int64),
+      Arrow::Field.new("float",  :float),
+      Arrow::Field.new("double", :double),
+    ]
+    schema = Arrow::Schema.new(fields)
+
+    tempfile = Tempfile.new(["batch", ".arrows"])
+    Arrow::FileOutputStream.open(tempfile.path, false) do |output|
+      Arrow::RecordBatchStreamWriter.open(output, schema) do |writer|
+        uints = [1, 2, 4, 8]
+        ints = [1, -2, 4, -8]
+        floats = [1.1, -2.2, 4.4, -8.8]
+        columns = [
+          Arrow::UInt8Array.new(uints),
+          Arrow::UInt16Array.new(uints),
+          Arrow::UInt32Array.new(uints),
+          Arrow::UInt64Array.new(uints),
+          Arrow::Int8Array.new(ints),
+          Arrow::Int16Array.new(ints),
+          Arrow::Int32Array.new(ints),
+          Arrow::Int64Array.new(ints),
+          Arrow::FloatArray.new(floats),
+          Arrow::DoubleArray.new(floats),
+        ]
+
+        record_batch = Arrow::RecordBatch.new(schema, 4, columns)
+        writer.write_record_batch(record_batch)
+      end
+    end
+
+    Arrow::MemoryMappedInputStream.open(tempfile.path) do |input|
+      reader = Arrow::RecordBatchStreamReader.new(input)
+      reader.each do |record_batch|
+        assert_equal([
+                       {
+                         "uint8"  => 1,
+                         "uint16" => 1,
+                         "uint32" => 1,
+                         "uint64" => 1,
+                         "int8"   => 1,
+                         "int16"  => 1,
+                         "int32"  => 1,
+                         "int64"  => 1,
+                         "float"  => 1.100000023841858,
+                         "double" => 1.1,
+                       },
+                       {
+                         "uint8"  => 2,
+                         "uint16" => 2,
+                         "uint32" => 2,
+                         "uint64" => 2,
+                         "int8"   => -2,
+                         "int16"  => -2,
+                         "int32"  => -2,
+                         "int64"  => -2,
+                         "float"  => -2.200000047683716,
+                         "double" => -2.2,
+                       },
+                       {
+                         "uint8"  => 4,
+                         "uint16" => 4,
+                         "uint32" => 4,
+                         "uint64" => 4,
+                         "int8"   => 4,
+                         "int16"  => 4,
+                         "int32"  => 4,
+                         "int64"  => 4,
+                         "float"  => 4.400000095367432,
+                         "double" => 4.4,
+                       },
+                       {
+                         "uint8"  => 8,
+                         "uint16" => 8,
+                         "uint32" => 8,
+                         "uint64" => 8,
+                         "int8"   => -8,
+                         "int16"  => -8,
+                         "int32"  => -8,
+                         "int64"  => -8,
+                         "float"  => -8.800000190734863,
+                         "double" => -8.8,
+                       },
+                     ],
+                     record_batch.collect(&:to_h))
+      end
+    end
+  end
+
+  sub_test_case("#each") do
+    test("without block") do
+      buffer = Arrow::ResizableBuffer.new(1024)
+      Arrow::Table.new(number: [1, 2, 3]).save(buffer, format: :arrows)
+      Arrow::BufferInputStream.open(buffer) do |input|
+        reader = Arrow::RecordBatchStreamReader.new(input)
+        assert_equal([
+                       Arrow::RecordBatch.new(number: [1, 2, 3]),
+                     ],
+                     reader.each.to_a)
+      end
+    end
+  end
+end

data/test/test-scalar.rb

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class ScalarTest < Test::Unit::TestCase
+  sub_test_case(".resolve") do
+    test("Scalar") do
+      assert_equal(Arrow::Int32Scalar.new(29),
+                   Arrow::Scalar.resolve(Arrow::Int32Scalar.new(29)))
+    end
+
+    test("true") do
+      assert_equal(Arrow::BooleanScalar.new(true),
+                   Arrow::Scalar.resolve(true))
+    end
+
+    test("false") do
+      assert_equal(Arrow::BooleanScalar.new(false),
+                   Arrow::Scalar.resolve(false))
+    end
+
+    test("Symbol") do
+      assert_equal(Arrow::StringScalar.new("hello"),
+                   Arrow::Scalar.resolve(:hello))
+    end
+
+    test("String") do
+      assert_equal(Arrow::StringScalar.new("hello"),
+                   Arrow::Scalar.resolve("hello"))
+    end
+
+    test("Integer") do
+      assert_equal(Arrow::Int64Scalar.new(-29),
+                   Arrow::Scalar.resolve(-29))
+    end
+
+    test("Float") do
+      assert_equal(Arrow::DoubleScalar.new(2.9),
+                   Arrow::Scalar.resolve(2.9))
+    end
+
+    test("Int64Scalar, :int32") do
+      assert_equal(Arrow::Int32Scalar.new(-29),
+                   Arrow::Scalar.resolve(Arrow::Int64Scalar.new(-29), :int32))
+    end
+
+    test("Integer, :int32") do
+      assert_equal(Arrow::Int32Scalar.new(-29),
+                   Arrow::Scalar.resolve(-29, :int32))
+    end
+  end
+end