red-arrow 8.0.0 → 24.0.0

data/test/test-record-batch-builder.rb

@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RecordBatchBuilderTest < Test::Unit::TestCase
-  sub_test_case(".new") do
-    test("Schema") do
-      schema = Arrow::Schema.new(visible: :boolean,
-                                 count: :uint32)
-      builder = Arrow::RecordBatchBuilder.new(schema)
-      assert_equal(schema,
-                   builder.schema)
-    end
-
-    test("Hash") do
-      builder = Arrow::RecordBatchBuilder.new(visible: :boolean,
-                                              count: :uint32)
-      assert_equal(Arrow::Schema.new(visible: :boolean,
-                                     count: :uint32),
-                   builder.schema)
-    end
-  end
-
-  sub_test_case("instance methods") do
-    def setup
-      @schema = Arrow::Schema.new(visible: :boolean,
-                                  count: :uint32)
-      @builder = Arrow::RecordBatchBuilder.new(@schema)
-    end
-
-    sub_test_case("#[]") do
-      test("String") do
-        assert_equal(Arrow::BooleanDataType.new,
-                     @builder["visible"].value_data_type)
-      end
-
-      test("Symbol") do
-        assert_equal(Arrow::BooleanDataType.new,
-                     @builder[:visible].value_data_type)
-      end
-
-      test("Integer") do
-        assert_equal(Arrow::UInt32DataType.new,
-                     @builder[1].value_data_type)
-      end
-    end
-
-    test("#append") do
-      records = [
-        {visible: true, count: 1},
-      ]
-      columns = {
-        visible: [false],
-        count: [2],
-      }
-      arrays = [
-        Arrow::BooleanArray.new([true, false]),
-        Arrow::UInt32Array.new([1, 2]),
-      ]
-      @builder.append(records, columns)
-      assert_equal(Arrow::RecordBatch.new(@schema,
-                                          arrays[0].length,
-                                          arrays),
-                   @builder.flush)
-    end
-
-    test("#append_records") do
-      records = [
-        {visible: true, count: 1},
-        {visible: true, count: 2, garbage: "garbage"},
-        {visible: true},
-        [false, 4],
-        nil,
-        [true],
-      ]
-      arrays = [
-        Arrow::BooleanArray.new([true, true, true, false, nil, true]),
-        Arrow::UInt32Array.new([1, 2, nil, 4, nil, nil]),
-      ]
-      @builder.append_records(records)
-      assert_equal(Arrow::RecordBatch.new(@schema,
-                                          arrays[0].length,
-                                          arrays),
-                   @builder.flush)
-    end
-
-    test("#append_columns") do
-      columns = {
-        visible: [true, true, true, false, nil, true],
-        count: [1, 2, nil, 4, nil, nil],
-      }
-      arrays = [
-        Arrow::BooleanArray.new(columns[:visible]),
-        Arrow::UInt32Array.new(columns[:count]),
-      ]
-      @builder.append_columns(columns)
-      assert_equal(Arrow::RecordBatch.new(@schema,
-                                          arrays[0].length,
-                                          arrays),
-                   @builder.flush)
-    end
-
-    test("#column_builders") do
-      column_builders = [
-        @builder.get_column_builder(0),
-        @builder.get_column_builder(1),
-      ]
-      assert_equal(column_builders,
-                   @builder.column_builders)
-    end
-  end
-end

data/test/test-record-batch-file-reader.rb

@@ -1,115 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RecordBatchFileReaderTest < Test::Unit::TestCase
-  test("write/read") do
-    fields = [
-      Arrow::Field.new("uint8",  :uint8),
-      Arrow::Field.new("uint16", :uint16),
-      Arrow::Field.new("uint32", :uint32),
-      Arrow::Field.new("uint64", :uint64),
-      Arrow::Field.new("int8",   :int8),
-      Arrow::Field.new("int16",  :int16),
-      Arrow::Field.new("int32",  :int32),
-      Arrow::Field.new("int64",  :int64),
-      Arrow::Field.new("float",  :float),
-      Arrow::Field.new("double", :double),
-    ]
-    schema = Arrow::Schema.new(fields)
-
-    tempfile = Tempfile.new(["batch", ".arrow"])
-    Arrow::FileOutputStream.open(tempfile.path, false) do |output|
-      Arrow::RecordBatchFileWriter.open(output, schema) do |writer|
-        uints = [1, 2, 4, 8]
-        ints = [1, -2, 4, -8]
-        floats = [1.1, -2.2, 4.4, -8.8]
-        columns = [
-          Arrow::UInt8Array.new(uints),
-          Arrow::UInt16Array.new(uints),
-          Arrow::UInt32Array.new(uints),
-          Arrow::UInt64Array.new(uints),
-          Arrow::Int8Array.new(ints),
-          Arrow::Int16Array.new(ints),
-          Arrow::Int32Array.new(ints),
-          Arrow::Int64Array.new(ints),
-          Arrow::FloatArray.new(floats),
-          Arrow::DoubleArray.new(floats),
-        ]
-
-        record_batch = Arrow::RecordBatch.new(schema, 4, columns)
-        writer.write_record_batch(record_batch)
-      end
-    end
-
-    Arrow::MemoryMappedInputStream.open(tempfile.path) do |input|
-      reader = Arrow::RecordBatchFileReader.new(input)
-      reader.each do |record_batch|
-        assert_equal([
-                       {
-                         "uint8"  => 1,
-                         "uint16" => 1,
-                         "uint32" => 1,
-                         "uint64" => 1,
-                         "int8"   => 1,
-                         "int16"  => 1,
-                         "int32"  => 1,
-                         "int64"  => 1,
-                         "float"  => 1.100000023841858,
-                         "double" => 1.1,
-                       },
-                       {
-                         "uint8"  => 2,
-                         "uint16" => 2,
-                         "uint32" => 2,
-                         "uint64" => 2,
-                         "int8"   => -2,
-                         "int16"  => -2,
-                         "int32"  => -2,
-                         "int64"  => -2,
-                         "float"  => -2.200000047683716,
-                         "double" => -2.2,
-                       },
-                       {
-                         "uint8"  => 4,
-                         "uint16" => 4,
-                         "uint32" => 4,
-                         "uint64" => 4,
-                         "int8"   => 4,
-                         "int16"  => 4,
-                         "int32"  => 4,
-                         "int64"  => 4,
-                         "float"  => 4.400000095367432,
-                         "double" => 4.4,
-                       },
-                       {
-                         "uint8"  => 8,
-                         "uint16" => 8,
-                         "uint32" => 8,
-                         "uint64" => 8,
-                         "int8"   => -8,
-                         "int16"  => -8,
-                         "int32"  => -8,
-                         "int64"  => -8,
-                         "float"  => -8.800000190734863,
-                         "double" => -8.8,
-                       },
-                     ],
-                     record_batch.collect(&:to_h))
-      end
-    end
-  end
-end

data/test/test-record-batch-iterator.rb

@@ -1,37 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RecordBatchIteratorTest < Test::Unit::TestCase
-  def setup
-    @schema = Arrow::Schema.new(visible: :boolean,
-                                count: :uint32)
-    @record_batches = [
-      Arrow::RecordBatch.new(@schema,
-                             visible: [true],
-                             count: [1]),
-      Arrow::RecordBatch.new(@schema,
-                             visible: [false, nil],
-                             count: [nil, 3]),
-    ]
-    @iterator = Arrow::RecordBatchIterator.new(@record_batches)
-  end
-
-  def test_to_a
-    assert_equal(@record_batches,
-                 @iterator.to_a)
-  end
-end

data/test/test-record-batch-reader.rb

@@ -1,46 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class TestRecordBatchReader < Test::Unit::TestCase
-  sub_test_case(".try_convert") do
-    test("Arrow::RecordBatch") do
-      record_batch =
-        Arrow::RecordBatch.new("count" => [1, 2, 3],
-                               "private" => [true, false, true])
-      reader = Arrow::RecordBatchReader.try_convert(record_batch)
-      assert_equal(record_batch,
-                   reader.read_next)
-    end
-
-    test("[Arrow::RecordBatch]") do
-      record_batch =
-        Arrow::RecordBatch.new("count" => [1, 2, 3],
-                               "private" => [true, false, true])
-      reader = Arrow::RecordBatchReader.try_convert([record_batch])
-      assert_equal(record_batch,
-                   reader.read_next)
-    end
-
-    test("Arrow::Table") do
-      table = Arrow::Table.new("count" => [1, 2, 3],
-                               "private" => [true, false, true])
-      reader = Arrow::RecordBatchReader.try_convert(table)
-      assert_equal(table,
-                   reader.read_all)
-    end
-  end
-end

data/test/test-record-batch.rb

@@ -1,182 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RecordBatchTest < Test::Unit::TestCase
-  sub_test_case(".new") do
-    def setup
-      @schema = Arrow::Schema.new(visible: :boolean,
-                                  count: :uint32)
-    end
-
-    test("[raw_table]") do
-      raw_table = {
-        visible: [true, nil, false],
-        count: [1, nil, 3],
-      }
-      record_batch = Arrow::RecordBatch.new(raw_table)
-      assert_equal([
-                     {"visible" => true,  "count" => 1},
-                     {"visible" => nil,   "count" => nil},
-                     {"visible" => false, "count" => 3},
-                   ],
-                   record_batch.each_record.collect(&:to_h))
-    end
-
-    test("[Schema, records]") do
-      records = [
-        {visible: true, count: 1},
-        nil,
-        [false, 3],
-      ]
-      record_batch = Arrow::RecordBatch.new(@schema, records)
-      assert_equal([
-                     {"visible" => true,  "count" => 1},
-                     {"visible" => nil,   "count" => nil},
-                     {"visible" => false, "count" => 3},
-                   ],
-                   record_batch.each_record.collect(&:to_h))
-    end
-
-    test("[Schema, columns]") do
-      columns = {
-        visible: [true, nil, false],
-        count: [1, 2, nil],
-      }
-      record_batch = Arrow::RecordBatch.new(@schema, columns)
-      assert_equal([
-                     {"visible" => true,  "count" => 1},
-                     {"visible" => nil,   "count" => 2},
-                     {"visible" => false, "count" => nil},
-                   ],
-                   record_batch.each_record.collect(&:to_h))
-    end
-
-    test("[Schema, n_rows, columns]") do
-      columns = [
-        Arrow::BooleanArray.new([true, nil, false]),
-        Arrow::UInt32Array.new([1, 2, nil]),
-      ]
-      n_rows = columns[0].length
-      record_batch = Arrow::RecordBatch.new(@schema, n_rows, columns)
-      assert_equal([
-                     {"visible" => true,  "count" => 1},
-                     {"visible" => nil,   "count" => 2},
-                     {"visible" => false, "count" => nil},
-                   ],
-                   record_batch.each_record.collect(&:to_h))
-    end
-  end
-
-  sub_test_case("instance methods") do
-    def setup
-      @schema = Arrow::Schema.new(count: :uint32)
-      @counts = Arrow::UInt32Array.new([1, 2, 4, 8])
-      @record_batch = Arrow::RecordBatch.new(@schema, @counts.length, [@counts])
-    end
-
-    sub_test_case("#each") do
-      test("default") do
-        records = []
-        @record_batch.each do |record|
-          records << [record, record.index]
-        end
-        assert_equal([
-                       [0, 0],
-                       [1, 1],
-                       [2, 2],
-                       [3, 3],
-                     ],
-                     records.collect {|record, i| [record.index, i]})
-      end
-
-      test("reuse_record: true") do
-        records = []
-        @record_batch.each(reuse_record: true) do |record|
-          records << [record, record.index]
-        end
-        assert_equal([
-                       [3, 0],
-                       [3, 1],
-                       [3, 2],
-                       [3, 3],
-                     ],
-                     records.collect {|record, i| [record.index, i]})
-      end
-    end
-
-    test("#to_table") do
-      assert_equal(Arrow::Table.new(@schema, [@counts]),
-                   @record_batch.to_table)
-    end
-
-    sub_test_case("#==") do
-      test("Arrow::RecordBatch") do
-        assert do
-          @record_batch == @record_batch
-        end
-      end
-
-      test("not Arrow::RecordBatch") do
-        assert do
-          not (@record_batch == 29)
-        end
-      end
-    end
-
-    sub_test_case("#[]") do
-      def setup
-        @record_batch = Arrow::RecordBatch.new(a: [true],
-                                               b: [true],
-                                               c: [true],
-                                               d: [true],
-                                               e: [true],
-                                               f: [true],
-                                               g: [true])
-      end
-
-      test("[String]") do
-        assert_equal(Arrow::Column.new(@record_batch, 0),
-                     @record_batch["a"])
-      end
-
-      test("[Symbol]") do
-        assert_equal(Arrow::Column.new(@record_batch, 1),
-                     @record_batch[:b])
-      end
-
-      test("[Integer]") do
-        assert_equal(Arrow::Column.new(@record_batch, 6),
-                     @record_batch[-1])
-      end
-
-      test("[Range]") do
-        assert_equal(Arrow::RecordBatch.new(d: [true],
-                                            e: [true]),
-                     @record_batch[3..4])
-      end
-
-      test("[[Symbol, String, Integer, Range]]") do
-        assert_equal(Arrow::RecordBatch.new(c: [true],
-                                            a: [true],
-                                            g: [true],
-                                            d: [true],
-                                            e: [true]),
-                     @record_batch[[:c, "a", -1, 3..4]])
-      end
-    end
-  end
-end

data/test/test-schema.rb

@@ -1,134 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class SchemaTest < Test::Unit::TestCase
-  include Helper::Omittable
-
-  def setup
-    @count_field = Arrow::Field.new("count", :uint32)
-    @visible_field = Arrow::Field.new("visible", :boolean)
-  end
-
-  sub_test_case(".new") do
-    test("[Arrow::Field]") do
-      fields = [
-        @count_field,
-        @visible_field,
-      ]
-      assert_equal("count: uint32\n" +
-                   "visible: bool",
-                   Arrow::Schema.new(fields).to_s)
-    end
-
-    test("[Arrow::Field, Hash]") do
-      fields = [
-        @count_field,
-        {name: "visible", type: :boolean},
-      ]
-      assert_equal("count: uint32\n" +
-                   "visible: bool",
-                   Arrow::Schema.new(fields).to_s)
-    end
-
-    test("{String, Symbol => Arrow::DataType}") do
-      fields = {
-        "count" => Arrow::UInt32DataType.new,
-        :visible => :boolean,
-      }
-      assert_equal("count: uint32\n" +
-                   "visible: bool",
-                   Arrow::Schema.new(fields).to_s)
-    end
-
-    test("{String, Symbol => Hash}") do
-      fields = {
-        "count" => {type: :uint32},
-        :tags => {
-          type: :list,
-          field: {
-            name: "tag",
-            type: :string,
-          },
-        },
-      }
-      assert_equal("count: uint32\n" +
-                   "tags: list<tag: string>",
-                   Arrow::Schema.new(fields).to_s)
-    end
-  end
-
-  sub_test_case("instance methods") do
-    def setup
-      super
-      @schema = Arrow::Schema.new([@count_field, @visible_field])
-    end
-
-    sub_test_case("#[]") do
-      test("[String]") do
-        assert_equal([@count_field, @visible_field],
-                     [@schema["count"], @schema["visible"]])
-      end
-
-      test("[Symbol]") do
-        assert_equal([@count_field, @visible_field],
-                     [@schema[:count], @schema[:visible]])
-      end
-
-      test("[Integer]") do
-        assert_equal([@count_field, @visible_field],
-                     [@schema[0], @schema[1]])
-      end
-
-      test("[invalid]") do
-        invalid = []
-        message = "field name or index must be String, Symbol or Integer"
-        message << ": <#{invalid.inspect}>"
-        assert_raise(ArgumentError.new(message)) do
-          @schema[invalid]
-        end
-      end
-    end
-
-    sub_test_case("#==") do
-      test("Arrow::Schema") do
-        assert do
-          @schema == @schema
-        end
-      end
-
-      test("not Arrow::Schema") do
-        assert do
-          not (@schema == 29)
-        end
-      end
-    end
-
-    sub_test_case("#to_s") do
-      test("show_metadata") do
-        require_gi_bindings(3, 4, 2)
-
-        schema = @schema.with_metadata("key" => "value")
-        assert_equal(<<-SCHEMA.chomp, schema.to_s(show_metadata: true))
-count: uint32
-visible: bool
--- metadata --
-key: value
-        SCHEMA
-      end
-    end
-  end
-end