red-arrow 0.11.0 → 0.12.0

data/test/test-dense-union-data-type.rb

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class DenseUnionDataTypeTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    def setup
+      @fields = [
+        Arrow::Field.new("visible", :boolean),
+        {
+          name: "count",
+          type: :int32,
+        },
+      ]
+    end
+
+    test("ordered arguments") do
+      assert_equal("union[dense]<visible: bool=2, count: int32=9>",
+                   Arrow::DenseUnionDataType.new(@fields, [2, 9]).to_s)
+    end
+
+    test("description") do
+      assert_equal("union[dense]<visible: bool=2, count: int32=9>",
+                   Arrow::DenseUnionDataType.new(fields: @fields,
+                                                  type_codes: [2, 9]).to_s)
+    end
+  end
+end

data/test/test-dictionary-data-type.rb

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class DictionaryDataTypeTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    def setup
+      @index_data_type = :int8
+      @dictionary = Arrow::StringArray.new(["Hello", "World"])
+      @ordered = true
+    end
+
+    test("ordered arguments") do
+      assert_equal("dictionary<values=string, indices=int8, ordered=1>",
+                   Arrow::DictionaryDataType.new(@index_data_type,
+                                                 @dictionary,
+                                                 @ordered).to_s)
+    end
+
+    test("description") do
+      assert_equal("dictionary<values=string, indices=int8, ordered=1>",
+                   Arrow::DictionaryDataType.new(index_data_type: @index_data_type,
+                                                 dictionary: @dictionary,
+                                                 ordered: @ordered).to_s)
+    end
+  end
+end

data/test/test-feather.rb

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class FeatherTest < Test::Unit::TestCase
+  include Helper::Fixture
+
+  def test_save_load
+    columns = {
+      "message" => Arrow::StringArray.new(["Start", "Crash", "Shutdown"]),
+      "is_critical" => Arrow::BooleanArray.new([false, true, false]),
+    }
+    table = Arrow::Table.new(columns)
+
+    output = Tempfile.new(["red-arrow", ".feather"])
+    table.save(output.path)
+    output.close
+
+    assert_equal(table, Arrow::Table.load(output.path))
+  end
+end

data/test/test-field.rb

@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class FieldTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("String, Arrow::DataType") do
+      assert_equal("visible: bool",
+                   Arrow::Field.new("visible", Arrow::BooleanDataType.new).to_s)
+    end
+
+    test("Symbol, Arrow::DataType") do
+      assert_equal("visible: bool",
+                   Arrow::Field.new(:visible, Arrow::BooleanDataType.new).to_s)
+    end
+
+    test("String, Symbol") do
+      assert_equal("visible: bool",
+                   Arrow::Field.new(:visible, :boolean).to_s)
+    end
+
+    test("String, Hash") do
+      assert_equal("visible: bool",
+                   Arrow::Field.new(:visible, type: :boolean).to_s)
+    end
+
+    test("description: String") do
+      assert_equal("visible: bool",
+                   Arrow::Field.new(name: "visible",
+                                    data_type: :boolean).to_s)
+    end
+
+    test("description: Symbol") do
+      assert_equal("visible: bool",
+                   Arrow::Field.new(name: :visible,
+                                    data_type: :boolean).to_s)
+    end
+
+    test("description: shortcut") do
+      assert_equal("visible: bool",
+                   Arrow::Field.new(name: :visible,
+                                    type: :boolean).to_s)
+    end
+
+    test("Hash: shortcut: additional") do
+      description = {
+        name: :tags,
+        type: :list,
+        field: {
+          name: "tag",
+          type: :string,
+        },
+      }
+      assert_equal("tags: list<tag: string>",
+                   Arrow::Field.new(description).to_s)
+    end
+  end
+end

data/test/test-file-output-stream.rb

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestFileOutputStream < Test::Unit::TestCase
+  sub_test_case(".open") do
+    def setup
+      @file = Tempfile.open("arrow-file-output-stream")
+      @file.write("Hello")
+      @file.close
+    end
+
+    def test_default
+      Arrow::FileOutputStream.open(@file.path) do |file|
+        file.write(" World")
+      end
+      assert_equal(" World", File.read(@file.path))
+    end
+
+    def test_options_append
+      Arrow::FileOutputStream.open(@file.path, append: true) do |file|
+        file.write(" World")
+      end
+      assert_equal("Hello World", File.read(@file.path))
+    end
+
+    def test_append_true
+      Arrow::FileOutputStream.open(@file.path, true) do |file|
+        file.write(" World")
+      end
+      assert_equal("Hello World", File.read(@file.path))
+    end
+
+    def test_append_false
+      Arrow::FileOutputStream.open(@file.path, false) do |file|
+        file.write(" World")
+      end
+      assert_equal(" World", File.read(@file.path))
+    end
+  end
+end

data/test/test-list-array-builder.rb

@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class ListArrayBuilderTest < Test::Unit::TestCase
+  def setup
+    @data_type = Arrow::ListDataType.new(name: "visible", type: :boolean)
+    @builder = Arrow::ListArrayBuilder.new(@data_type)
+  end
+
+  sub_test_case("#append_value") do
+    test("nil") do
+      @builder.append_value(nil)
+      array = @builder.finish
+      assert_equal(nil, array[0])
+    end
+
+    test("Array") do
+      @builder.append_value([true, false, true])
+      array = @builder.finish
+      assert_equal([true, false, true], array[0].to_a)
+    end
+  end
+
+  sub_test_case("#append_values") do
+    test("[nil, Array]") do
+      @builder.append_values([[false], nil, [true, false, true]])
+      array = @builder.finish
+      assert_equal([
+                     [false],
+                     nil,
+                     [true, false, true],
+                   ],
+                   array.collect {|list| list ? list.to_a : nil})
+    end
+
+    test("is_valids") do
+      @builder.append_values([[false], [true, true], [true, false, true]],
+                             [true, false, true])
+      array = @builder.finish
+      assert_equal([
+                     [false],
+                     nil,
+                     [true, false, true],
+                   ],
+                   array.collect {|list| list ? list.to_a : nil})
+    end
+  end
+
+  sub_test_case("#append") do
+    test("backward compatibility") do
+      @builder.append
+      @builder.value_builder.append(true)
+      @builder.value_builder.append(false)
+      @builder.append
+      @builder.value_builder.append(true)
+      array = @builder.finish
+
+      assert_equal([
+                     [true, false],
+                     [true],
+                   ],
+                   array.collect(&:to_a))
+    end
+  end
+end

data/test/test-list-array.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class ListArrayTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("build") do
+      data_type = Arrow::ListDataType.new(name: "visible", type: :boolean)
+      values = [
+        [true, false],
+        nil,
+        [false, true, false],
+      ]
+      array = Arrow::ListArray.new(data_type, values)
+      assert_equal(values,
+                   array.collect {|value| value ? value.to_a : nil})
+    end
+  end
+end

data/test/test-list-data-type.rb

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class ListDataTypeTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("Arrow::Field") do
+      field = Arrow::Field.new(:tag, :string)
+      assert_equal("list<tag: string>",
+                   Arrow::ListDataType.new(field).to_s)
+    end
+
+    test("Hash") do
+      assert_equal("list<tag: string>",
+                   Arrow::ListDataType.new(name: "tag", type: :string).to_s)
+    end
+
+    test("field: Arrow::Field") do
+      field = Arrow::Field.new(:tag, :string)
+      assert_equal("list<tag: string>",
+                   Arrow::ListDataType.new(field: field).to_s)
+    end
+
+    test("field: Hash") do
+      field_description = {name: "tag", type: :string}
+      assert_equal("list<tag: string>",
+                   Arrow::ListDataType.new(field: field_description).to_s)
+    end
+  end
+end

data/test/test-record-batch-builder.rb

@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class RecordBatchBuilderTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("Schema") do
+      schema = Arrow::Schema.new(visible: :boolean,
+                                 count: :uint32)
+      builder = Arrow::RecordBatchBuilder.new(schema)
+      assert_equal(schema,
+                   builder.schema)
+    end
+
+    test("Hash") do
+      builder = Arrow::RecordBatchBuilder.new(visible: :boolean,
+                                              count: :uint32)
+      assert_equal(Arrow::Schema.new(visible: :boolean,
+                                     count: :uint32),
+                   builder.schema)
+    end
+  end
+
+  sub_test_case("instance methods") do
+    def setup
+      @schema = Arrow::Schema.new(visible: :boolean,
+                                  count: :uint32)
+      @builder = Arrow::RecordBatchBuilder.new(@schema)
+    end
+
+    sub_test_case("#[]") do
+      test("String") do
+        assert_equal(Arrow::BooleanDataType.new,
+                     @builder["visible"].value_data_type)
+      end
+
+      test("Symbol") do
+        assert_equal(Arrow::BooleanDataType.new,
+                     @builder[:visible].value_data_type)
+      end
+
+      test("Integer") do
+        assert_equal(Arrow::UInt32DataType.new,
+                     @builder[1].value_data_type)
+      end
+    end
+
+    test("#append") do
+      records = [
+        {visible: true, count: 1},
+      ]
+      columns = {
+        visible: [false],
+        count: [2],
+      }
+      arrays = [
+        Arrow::BooleanArray.new([true, false]),
+        Arrow::UInt32Array.new([1, 2]),
+      ]
+      @builder.append(records, columns)
+      assert_equal(Arrow::RecordBatch.new(@schema,
+                                          arrays[0].length,
+                                          arrays),
+                   @builder.flush)
+    end
+
+    test("#append_records") do
+      records = [
+        {visible: true, count: 1},
+        {visible: true, count: 2, garbage: "garbage"},
+        {visible: true},
+        [false, 4],
+        nil,
+        [true],
+      ]
+      arrays = [
+        Arrow::BooleanArray.new([true, true, true, false, nil, true]),
+        Arrow::UInt32Array.new([1, 2, nil, 4, nil, nil]),
+      ]
+      @builder.append_records(records)
+      assert_equal(Arrow::RecordBatch.new(@schema,
+                                          arrays[0].length,
+                                          arrays),
+                   @builder.flush)
+    end
+
+    test("#append_columns") do
+      columns = {
+        visible: [true, true, true, false, nil, true],
+        count: [1, 2, nil, 4, nil, nil],
+      }
+      arrays = [
+        Arrow::BooleanArray.new(columns[:visible]),
+        Arrow::UInt32Array.new(columns[:count]),
+      ]
+      @builder.append_columns(columns)
+      assert_equal(Arrow::RecordBatch.new(@schema,
+                                          arrays[0].length,
+                                          arrays),
+                   @builder.flush)
+    end
+  end
+end