red-arrow 5.0.0 → 6.0.0

data/test/raw-records/test-map-array.rb

@@ -0,0 +1,441 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module RawRecordsMapArrayTests
+  def build_schema(type)
+    {
+      column: {
+        type: :map,
+        key: :string,
+        item: type
+      },
+    }
+  end
+
+  def test_null
+    records = [
+      [{"key1" => nil}],
+      [nil],
+    ]
+    target = build(:null, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_boolean
+    records = [
+      [{"key1" => true, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:boolean, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_int8
+    records = [
+      [{"key1" => -(2 ** 7), "key2" => nil}],
+      [nil],
+    ]
+    target = build(:int8, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_uint8
+    records = [
+      [{"key1" => (2 ** 8) - 1, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:uint8, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_int16
+    records = [
+      [{"key1" => -(2 ** 15), "key2" => nil}],
+      [nil],
+    ]
+    target = build(:int16, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_uint16
+    records = [
+      [{"key1" => (2 ** 16) - 1, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:uint16, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_int32
+    records = [
+      [{"key1" => -(2 ** 31), "key2" => nil}],
+      [nil],
+    ]
+    target = build(:int32, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_uint32
+    records = [
+      [{"key1" => (2 ** 32) - 1, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:uint32, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_int64
+    records = [
+      [{"key1" => -(2 ** 63), "key2" => nil}],
+      [nil],
+    ]
+    target = build(:int64, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_uint64
+    records = [
+      [{"key1" => (2 ** 64) - 1, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:uint64, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_float
+    records = [
+      [{"key1" => -1.0, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:float, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_double
+    records = [
+      [{"key1" => -1.0, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:double, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_binary
+    records = [
+      [{"key1" => "\xff".b, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:binary, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_string
+    records = [
+      [{"key1" => "Ruby", "key2" => nil}],
+      [nil],
+    ]
+    target = build(:string, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_date32
+    records = [
+      [{"key1" => Date.new(1960, 1, 1), "key2" => nil}],
+      [nil],
+    ]
+    target = build(:date32, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_date64
+    records = [
+      [{"key1" => DateTime.new(1960, 1, 1, 2, 9, 30), "key2" => nil}],
+      [nil],
+    ]
+    target = build(:date64, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_timestamp_second
+    records = [
+      [{"key1" => Time.parse("1960-01-01T02:09:30Z"), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :timestamp,
+                     unit: :second,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_timestamp_milli
+    records = [
+      [{"key1" => Time.parse("1960-01-01T02:09:30.123Z"), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :timestamp,
+                     unit: :milli,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_timestamp_micro
+    records = [
+      [{"key1" => Time.parse("1960-01-01T02:09:30.123456Z"), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :timestamp,
+                     unit: :micro,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_timestamp_nano
+    records = [
+      [{"key1" => Time.parse("1960-01-01T02:09:30.123456789Z"), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :timestamp,
+                     unit: :nano,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_time32_second
+    unit = Arrow::TimeUnit::SECOND
+    records = [
+      # 00:10:00
+      [{"key1" => Arrow::Time.new(unit, 60 * 10), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :time32,
+                     unit: :second,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_time32_milli
+    unit = Arrow::TimeUnit::MILLI
+    records = [
+      # 00:10:00.123
+      [{"key1" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :time32,
+                     unit: :milli,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_time64_micro
+    unit = Arrow::TimeUnit::MICRO
+    records = [
+      # 00:10:00.123456
+      [{"key1" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :time64,
+                     unit: :micro,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_time64_nano
+    unit = Arrow::TimeUnit::NANO
+    records = [
+      # 00:10:00.123456789
+      [{"key1" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :time64,
+                     unit: :nano,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_decimal128
+    records = [
+      [{"key1" => BigDecimal("92.92"), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :decimal128,
+                     precision: 8,
+                     scale: 2,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_decimal256
+    records = [
+      [{"key1" => BigDecimal("92.92"), "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :decimal256,
+                     precision: 38,
+                     scale: 2,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_list
+    records = [
+      [{"key1" => [true, nil, false], "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :list,
+                     field: {
+                       name: :element,
+                       type: :boolean,
+                     },
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_struct
+    records = [
+      [{"key1" => {"field" => true}, "key2" => nil, "key3" => {"field" => nil}}],
+      [nil],
+    ]
+    target = build({
+                     type: :struct,
+                     fields: [
+                       {
+                         name: :field,
+                         type: :boolean,
+                       },
+                     ],
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_map
+    records = [
+      [{"key1" => {"sub_key1" => true, "sub_key2" => nil}, "key2" => nil}],
+      [nil],
+    ]
+    target = build({
+                     type: :map,
+                     key: :string,
+                     item: :boolean,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_sparse_union
+    omit("Need to add support for SparseUnionArrayBuilder")
+    records = [
+      [{"key1" => {"field" => true, "key2" => nil, "key3" => {"field" => nil}}}],
+      [nil],
+    ]
+    target = build({
+                     type: :sparse_union,
+                     fields: [
+                       {
+                         name: :field1,
+                         type: :boolean,
+                       },
+                       {
+                         name: :field2,
+                         type: :uint8,
+                       },
+                     ],
+                     type_codes: [0, 1],
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_dense_union
+    omit("Need to add support for DenseUnionArrayBuilder")
+    records = [
+      [{"key1" => {"field1" => true}, "key2" => nil, "key3" => {"field2" => nil}}],
+      [nil],
+    ]
+    target = build({
+                     type: :dense_union,
+                     fields: [
+                       {
+                         name: :field1,
+                         type: :boolean,
+                       },
+                       {
+                         name: :field2,
+                         type: :uint8,
+                       },
+                     ],
+                     type_codes: [0, 1],
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_dictionary
+    omit("Need to add support for DictionaryArrayBuilder")
+    records = [
+      [{"key1" => "Ruby", "key2" => nil, "key3" => "GLib"}],
+      [nil],
+    ]
+    dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
+    target = build({
+                     type: :dictionary,
+                     index_data_type: :int8,
+                     dictionary: dictionary,
+                     ordered: true,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+end
+
+class RawRecordsRecordBatchMapArrayTest < Test::Unit::TestCase
+  include RawRecordsMapArrayTests
+
+  def build(type, records)
+    Arrow::RecordBatch.new(build_schema(type), records)
+  end
+end
+
+class RawRecordsTableMapArrayTest < Test::Unit::TestCase
+  include RawRecordsMapArrayTests
+
+  def build(type, records)
+    Arrow::Table.new(build_schema(type), records)
+  end
+end

data/test/raw-records/test-sparse-union-array.rb

@@ -384,6 +384,20 @@ module RawRecordsSparseUnionArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_map
+    records = [
+      [{"0" => {"key1" => true, "key2" => nil}}],
+      [{"1" => nil}],
+    ]
+    target = build({
+                     type: :map,
+                     key: :string,
+                     item: :boolean,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_sparse_union
     omit("Need to add support for SparseUnionArrayBuilder")
     records = [

data/test/raw-records/test-struct-array.rb

@@ -381,6 +381,21 @@ module RawRecordsStructArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_map
+    records = [
+      [{"field" => {"key1" => true, "key2" => nil}}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :map,
+                     key: :string,
+                     item: :boolean,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_sparse_union
     omit("Need to add support for SparseUnionArrayBuilder")
     records = [

data/test/test-array-builder.rb

@@ -28,6 +28,13 @@ class ArrayBuilderTest < Test::Unit::TestCase
                      ["Hello", nil, "World"])
       end
 
+      test("symbols") do
+        array = Arrow::ArrayBuilder.build([:hello, nil, :world])
+        expected_builder = Arrow::StringDictionaryArrayBuilder.new
+        assert_equal(expected_builder.build(["hello", nil, "world"]),
+                     array)
+      end
+
       test("boolean") do
         assert_build(Arrow::ArrayBuilder,
                      [true, nil, false])

data/test/test-binary-dictionary-array-builder.rb

@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class BinaryDictionaryArrayBuilderTest < Test::Unit::TestCase
+  def setup
+    @builder = Arrow::BinaryDictionaryArrayBuilder.new
+  end
+
+  sub_test_case("#append_values") do
+    test("[nil]") do
+      @builder.append_values([nil])
+      array = @builder.finish
+      assert_equal([
+                     [],
+                     [nil],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+
+    test("[String]") do
+      @builder.append_values(["he\xffllo"])
+      array = @builder.finish
+      assert_equal([
+                     ["he\xffllo".b],
+                     [0],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+
+    test("[Symbol]") do
+      @builder.append_values([:hello])
+      array = @builder.finish
+      assert_equal([
+                     ["hello"],
+                     [0],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+
+    test("[nil, String, Symbol]") do
+      @builder.append_values([
+                               nil,
+                               "He\xffllo",
+                               :world,
+                               "world",
+                             ])
+      array = @builder.finish
+      assert_equal([
+                     ["He\xffllo".b, "world"],
+                     [nil, 0, 1, 1],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+
+    test("is_valids") do
+      @builder.append_values([
+                               "He\xffllo",
+                               :world,
+                               :goodbye,
+                             ],
+                             [
+                               true,
+                               false,
+                               true,
+                             ])
+      array = @builder.finish
+      assert_equal([
+                     ["He\xffllo".b, "goodbye"],
+                     [0, nil, 1],
+                   ],
+                   [
+                     array.dictionary.to_a,
+                     array.indices.to_a,
+                   ])
+    end
+  end
+end

data/test/test-csv-loader.rb

@@ -86,20 +86,20 @@ class CSVLoaderTest < Test::Unit::TestCase
     test("null: with double quote") do
       path = fixture_path("null-with-double-quote.csv").to_s
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	name	score
-0	alice	   10
-1	bob 	     
-2	chris	   -1
+	name	 score
+0	alice	    10
+1	bob 	(null)
+2	chris	    -1
       TABLE
     end
 
     test("null: without double quote") do
       path = fixture_path("null-without-double-quote.csv").to_s
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	name	score
-0	alice	   10
-1	bob 	     
-2	chris	   -1
+	name	 score
+0	alice	    10
+1	bob 	(null)
+2	chris	    -1
       TABLE
     end
 

data/test/test-expression.rb

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestExpression < Test::Unit::TestCase
+  sub_test_case(".try_convert") do
+    test("Symbol") do
+      assert_equal(Arrow::FieldExpression.new("visible"),
+                   Arrow::Expression.try_convert(:visible))
+    end
+
+    test("[String]") do
+      assert_equal(Arrow::CallExpression.new("func", []),
+                   Arrow::Expression.try_convert(["func"]))
+    end
+
+    test("[Symbol]") do
+      assert_equal(Arrow::CallExpression.new("func", []),
+                   Arrow::Expression.try_convert([:func]))
+    end
+
+    test("[String, String]") do
+      assert_equal(Arrow::CallExpression.new("func", ["argument1"]),
+                   Arrow::Expression.try_convert(["func", "argument1"]))
+    end
+  end
+end