red-arrow 18.1.0 → 19.0.0

data/test/test-orc.rb

@@ -1,173 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class ORCTest < Test::Unit::TestCase
-  include Helper::Fixture
-
-  def setup
-    omit("Require Apache Arrow ORC") unless Arrow.const_defined?(:ORCFileReader)
-    @orc_path = fixture_path("TestOrcFile.test1.orc")
-  end
-
-  def pp_values(values)
-    "[\n  " + values.collect(&:inspect).join(",\n  ") + "\n]"
-  end
-
-  sub_test_case("load") do
-    test("default") do
-      table = Arrow::Table.load(@orc_path)
-      dump = table.columns.collect do |column|
-        [
-          column.field.to_s,
-          column.data.chunks.collect(&:to_s),
-        ]
-      end
-      assert_equal([
-                     ["boolean1: bool", [pp_values([false, true])]],
-                     ["byte1: int8", [pp_values([1, 100])]],
-                     ["short1: int16", [pp_values([1024, 2048])]],
-                     ["int1: int32", [pp_values([65536, 65536])]],
-                     [
-                       "long1: int64",
-                       [pp_values([9223372036854775807, 9223372036854775807])],
-                     ],
-                     ["float1: float", [pp_values([1, 2])]],
-                     ["double1: double", [pp_values([-15, -5])]],
-                     ["bytes1: binary", ["[\n  0001020304,\n  \n]"]],
-                     ["string1: string", [pp_values(["hi", "bye"])]],
-                     [
-                       "middle: " +
-                       "struct<list: " +
-                       "list<item: struct<int1: int32, string1: string>>>",
-                       [
-                         <<-STRUCT.chomp
--- is_valid: all not null
--- child 0 type: list<item: struct<int1: int32, string1: string>>
-  [
-    -- is_valid: all not null
-    -- child 0 type: int32
-      [
-        1,
-        2
-      ]
-    -- child 1 type: string
-      [
-        "bye",
-        "sigh"
-      ],
-    -- is_valid: all not null
-    -- child 0 type: int32
-      [
-        1,
-        2
-      ]
-    -- child 1 type: string
-      [
-        "bye",
-        "sigh"
-      ]
-  ]
-                          STRUCT
-                       ]
-                     ],
-                     [
-                       "list: list<item: struct<int1: int32, string1: string>>",
-                       [
-                         <<-LIST.chomp
-[
-  -- is_valid: all not null
-  -- child 0 type: int32
-    [
-      3,
-      4
-    ]
-  -- child 1 type: string
-    [
-      "good",
-      "bad"
-    ],
-  -- is_valid: all not null
-  -- child 0 type: int32
-    [
-      100000000,
-      -100000,
-      1234
-    ]
-  -- child 1 type: string
-    [
-      "cat",
-      "in",
-      "hat"
-    ]
-]
-                         LIST
-                       ]
-                     ],
-                     [
-                       "map: map<string, struct<int1: int32, string1: string>>",
-                       [
-                         <<-MAP.chomp
-[
-  keys:
-  []
-  values:
-  -- is_valid: all not null
-  -- child 0 type: int32
-    []
-  -- child 1 type: string
-    [],
-  keys:
-  [
-    "chani",
-    "mauddib"
-  ]
-  values:
-  -- is_valid: all not null
-  -- child 0 type: int32
-    [
-      5,
-      1
-    ]
-  -- child 1 type: string
-    [
-      "chani",
-      "mauddib"
-    ]
-]
-                         MAP
-                       ],
-                     ],
-                   ],
-                   dump)
-    end
-
-    test(":field_indexes") do
-      table = Arrow::Table.load(@orc_path, field_indexes: [1, 3])
-      dump = table.columns.collect do |column|
-        [
-          column.field.to_s,
-          column.data.chunks.collect(&:to_s),
-        ]
-      end
-      assert_equal([
-                     ["byte1: int8", [pp_values([1, 100])]],
-                     ["int1: int32", [pp_values([65536, 65536])]],
-                   ],
-                   dump)
-    end
-  end
-end

data/test/test-ractor.rb

@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RactorTest < Test::Unit::TestCase
-  include Helper::Omittable
-
-  ractor
-  test("ChunkedArray") do
-    require_ruby(3, 1, 0)
-    array = Arrow::Array.new([1, 2, 3])
-    chunked_array = Arrow::ChunkedArray.new([array])
-    Ractor.make_shareable(chunked_array)
-    ractor = Ractor.new do
-      recived_chunked_array = Ractor.receive
-      recived_chunked_array.chunks
-    end
-    ractor.send(chunked_array)
-    assert_equal([array], ractor.take)
-  end
-end

data/test/test-record-batch-builder.rb

@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RecordBatchBuilderTest < Test::Unit::TestCase
-  sub_test_case(".new") do
-    test("Schema") do
-      schema = Arrow::Schema.new(visible: :boolean,
-                                 count: :uint32)
-      builder = Arrow::RecordBatchBuilder.new(schema)
-      assert_equal(schema,
-                   builder.schema)
-    end
-
-    test("Hash") do
-      builder = Arrow::RecordBatchBuilder.new(visible: :boolean,
-                                              count: :uint32)
-      assert_equal(Arrow::Schema.new(visible: :boolean,
-                                     count: :uint32),
-                   builder.schema)
-    end
-  end
-
-  sub_test_case("instance methods") do
-    def setup
-      @schema = Arrow::Schema.new(visible: :boolean,
-                                  count: :uint32)
-      @builder = Arrow::RecordBatchBuilder.new(@schema)
-    end
-
-    sub_test_case("#[]") do
-      test("String") do
-        assert_equal(Arrow::BooleanDataType.new,
-                     @builder["visible"].value_data_type)
-      end
-
-      test("Symbol") do
-        assert_equal(Arrow::BooleanDataType.new,
-                     @builder[:visible].value_data_type)
-      end
-
-      test("Integer") do
-        assert_equal(Arrow::UInt32DataType.new,
-                     @builder[1].value_data_type)
-      end
-    end
-
-    test("#append") do
-      records = [
-        {visible: true, count: 1},
-      ]
-      columns = {
-        visible: [false],
-        count: [2],
-      }
-      arrays = [
-        Arrow::BooleanArray.new([true, false]),
-        Arrow::UInt32Array.new([1, 2]),
-      ]
-      @builder.append(records, columns)
-      assert_equal(Arrow::RecordBatch.new(@schema,
-                                          arrays[0].length,
-                                          arrays),
-                   @builder.flush)
-    end
-
-    test("#append_records") do
-      records = [
-        {visible: true, count: 1},
-        {visible: true, count: 2, garbage: "garbage"},
-        {visible: true},
-        [false, 4],
-        nil,
-        [true],
-      ]
-      arrays = [
-        Arrow::BooleanArray.new([true, true, true, false, nil, true]),
-        Arrow::UInt32Array.new([1, 2, nil, 4, nil, nil]),
-      ]
-      @builder.append_records(records)
-      assert_equal(Arrow::RecordBatch.new(@schema,
-                                          arrays[0].length,
-                                          arrays),
-                   @builder.flush)
-    end
-
-    test("#append_columns") do
-      columns = {
-        visible: [true, true, true, false, nil, true],
-        count: [1, 2, nil, 4, nil, nil],
-      }
-      arrays = [
-        Arrow::BooleanArray.new(columns[:visible]),
-        Arrow::UInt32Array.new(columns[:count]),
-      ]
-      @builder.append_columns(columns)
-      assert_equal(Arrow::RecordBatch.new(@schema,
-                                          arrays[0].length,
-                                          arrays),
-                   @builder.flush)
-    end
-
-    test("#column_builders") do
-      column_builders = [
-        @builder.get_column_builder(0),
-        @builder.get_column_builder(1),
-      ]
-      assert_equal(column_builders,
-                   @builder.column_builders)
-    end
-  end
-end

data/test/test-record-batch-file-reader.rb

@@ -1,136 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RecordBatchFileReaderTest < Test::Unit::TestCase
-  test("write/read") do
-    fields = [
-      Arrow::Field.new("uint8",  :uint8),
-      Arrow::Field.new("uint16", :uint16),
-      Arrow::Field.new("uint32", :uint32),
-      Arrow::Field.new("uint64", :uint64),
-      Arrow::Field.new("int8",   :int8),
-      Arrow::Field.new("int16",  :int16),
-      Arrow::Field.new("int32",  :int32),
-      Arrow::Field.new("int64",  :int64),
-      Arrow::Field.new("float",  :float),
-      Arrow::Field.new("double", :double),
-    ]
-    schema = Arrow::Schema.new(fields)
-
-    tempfile = Tempfile.new(["batch", ".arrow"])
-    Arrow::FileOutputStream.open(tempfile.path, false) do |output|
-      Arrow::RecordBatchFileWriter.open(output, schema) do |writer|
-        uints = [1, 2, 4, 8]
-        ints = [1, -2, 4, -8]
-        floats = [1.1, -2.2, 4.4, -8.8]
-        columns = [
-          Arrow::UInt8Array.new(uints),
-          Arrow::UInt16Array.new(uints),
-          Arrow::UInt32Array.new(uints),
-          Arrow::UInt64Array.new(uints),
-          Arrow::Int8Array.new(ints),
-          Arrow::Int16Array.new(ints),
-          Arrow::Int32Array.new(ints),
-          Arrow::Int64Array.new(ints),
-          Arrow::FloatArray.new(floats),
-          Arrow::DoubleArray.new(floats),
-        ]
-
-        record_batch = Arrow::RecordBatch.new(schema, 4, columns)
-        writer.write_record_batch(record_batch)
-      end
-    end
-
-    Arrow::MemoryMappedInputStream.open(tempfile.path) do |input|
-      reader = Arrow::RecordBatchFileReader.new(input)
-      reader.each do |record_batch|
-        assert_equal([
-                       {
-                         "uint8"  => 1,
-                         "uint16" => 1,
-                         "uint32" => 1,
-                         "uint64" => 1,
-                         "int8"   => 1,
-                         "int16"  => 1,
-                         "int32"  => 1,
-                         "int64"  => 1,
-                         "float"  => 1.100000023841858,
-                         "double" => 1.1,
-                       },
-                       {
-                         "uint8"  => 2,
-                         "uint16" => 2,
-                         "uint32" => 2,
-                         "uint64" => 2,
-                         "int8"   => -2,
-                         "int16"  => -2,
-                         "int32"  => -2,
-                         "int64"  => -2,
-                         "float"  => -2.200000047683716,
-                         "double" => -2.2,
-                       },
-                       {
-                         "uint8"  => 4,
-                         "uint16" => 4,
-                         "uint32" => 4,
-                         "uint64" => 4,
-                         "int8"   => 4,
-                         "int16"  => 4,
-                         "int32"  => 4,
-                         "int64"  => 4,
-                         "float"  => 4.400000095367432,
-                         "double" => 4.4,
-                       },
-                       {
-                         "uint8"  => 8,
-                         "uint16" => 8,
-                         "uint32" => 8,
-                         "uint64" => 8,
-                         "int8"   => -8,
-                         "int16"  => -8,
-                         "int32"  => -8,
-                         "int64"  => -8,
-                         "float"  => -8.800000190734863,
-                         "double" => -8.8,
-                       },
-                     ],
-                     record_batch.collect(&:to_h))
-      end
-    end
-  end
-
-  sub_test_case("#each") do
-    test("without block") do
-      buffer = Arrow::ResizableBuffer.new(1024)
-      Arrow::Table.new(number: [1, 2, 3]).save(buffer)
-      Arrow::BufferInputStream.open(buffer) do |input|
-        reader = Arrow::RecordBatchFileReader.new(input)
-        each = reader.each
-        assert_equal({
-                       size: 1,
-                       to_a: [
-                         Arrow::RecordBatch.new(number: [1, 2, 3]),
-                       ],
-                     },
-                     {
-                       size: each.size,
-                       to_a: each.to_a,
-                     })
-      end
-    end
-  end
-end

data/test/test-record-batch-iterator.rb

@@ -1,37 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RecordBatchIteratorTest < Test::Unit::TestCase
-  def setup
-    @schema = Arrow::Schema.new(visible: :boolean,
-                                count: :uint32)
-    @record_batches = [
-      Arrow::RecordBatch.new(@schema,
-                             visible: [true],
-                             count: [1]),
-      Arrow::RecordBatch.new(@schema,
-                             visible: [false, nil],
-                             count: [nil, 3]),
-    ]
-    @iterator = Arrow::RecordBatchIterator.new(@record_batches)
-  end
-
-  def test_to_a
-    assert_equal(@record_batches,
-                 @iterator.to_a)
-  end
-end

data/test/test-record-batch-reader.rb

@@ -1,46 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class TestRecordBatchReader < Test::Unit::TestCase
-  sub_test_case(".try_convert") do
-    test("Arrow::RecordBatch") do
-      record_batch =
-        Arrow::RecordBatch.new("count" => [1, 2, 3],
-                               "private" => [true, false, true])
-      reader = Arrow::RecordBatchReader.try_convert(record_batch)
-      assert_equal(record_batch,
-                   reader.read_next)
-    end
-
-    test("[Arrow::RecordBatch]") do
-      record_batch =
-        Arrow::RecordBatch.new("count" => [1, 2, 3],
-                               "private" => [true, false, true])
-      reader = Arrow::RecordBatchReader.try_convert([record_batch])
-      assert_equal(record_batch,
-                   reader.read_next)
-    end
-
-    test("Arrow::Table") do
-      table = Arrow::Table.new("count" => [1, 2, 3],
-                               "private" => [true, false, true])
-      reader = Arrow::RecordBatchReader.try_convert(table)
-      assert_equal(table,
-                   reader.read_all)
-    end
-  end
-end

data/test/test-record-batch-stream-reader.rb

@@ -1,129 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-class RecordBatchStreamReaderTest < Test::Unit::TestCase
-  test("write/read") do
-    fields = [
-      Arrow::Field.new("uint8",  :uint8),
-      Arrow::Field.new("uint16", :uint16),
-      Arrow::Field.new("uint32", :uint32),
-      Arrow::Field.new("uint64", :uint64),
-      Arrow::Field.new("int8",   :int8),
-      Arrow::Field.new("int16",  :int16),
-      Arrow::Field.new("int32",  :int32),
-      Arrow::Field.new("int64",  :int64),
-      Arrow::Field.new("float",  :float),
-      Arrow::Field.new("double", :double),
-    ]
-    schema = Arrow::Schema.new(fields)
-
-    tempfile = Tempfile.new(["batch", ".arrows"])
-    Arrow::FileOutputStream.open(tempfile.path, false) do |output|
-      Arrow::RecordBatchStreamWriter.open(output, schema) do |writer|
-        uints = [1, 2, 4, 8]
-        ints = [1, -2, 4, -8]
-        floats = [1.1, -2.2, 4.4, -8.8]
-        columns = [
-          Arrow::UInt8Array.new(uints),
-          Arrow::UInt16Array.new(uints),
-          Arrow::UInt32Array.new(uints),
-          Arrow::UInt64Array.new(uints),
-          Arrow::Int8Array.new(ints),
-          Arrow::Int16Array.new(ints),
-          Arrow::Int32Array.new(ints),
-          Arrow::Int64Array.new(ints),
-          Arrow::FloatArray.new(floats),
-          Arrow::DoubleArray.new(floats),
-        ]
-
-        record_batch = Arrow::RecordBatch.new(schema, 4, columns)
-        writer.write_record_batch(record_batch)
-      end
-    end
-
-    Arrow::MemoryMappedInputStream.open(tempfile.path) do |input|
-      reader = Arrow::RecordBatchStreamReader.new(input)
-      reader.each do |record_batch|
-        assert_equal([
-                       {
-                         "uint8"  => 1,
-                         "uint16" => 1,
-                         "uint32" => 1,
-                         "uint64" => 1,
-                         "int8"   => 1,
-                         "int16"  => 1,
-                         "int32"  => 1,
-                         "int64"  => 1,
-                         "float"  => 1.100000023841858,
-                         "double" => 1.1,
-                       },
-                       {
-                         "uint8"  => 2,
-                         "uint16" => 2,
-                         "uint32" => 2,
-                         "uint64" => 2,
-                         "int8"   => -2,
-                         "int16"  => -2,
-                         "int32"  => -2,
-                         "int64"  => -2,
-                         "float"  => -2.200000047683716,
-                         "double" => -2.2,
-                       },
-                       {
-                         "uint8"  => 4,
-                         "uint16" => 4,
-                         "uint32" => 4,
-                         "uint64" => 4,
-                         "int8"   => 4,
-                         "int16"  => 4,
-                         "int32"  => 4,
-                         "int64"  => 4,
-                         "float"  => 4.400000095367432,
-                         "double" => 4.4,
-                       },
-                       {
-                         "uint8"  => 8,
-                         "uint16" => 8,
-                         "uint32" => 8,
-                         "uint64" => 8,
-                         "int8"   => -8,
-                         "int16"  => -8,
-                         "int32"  => -8,
-                         "int64"  => -8,
-                         "float"  => -8.800000190734863,
-                         "double" => -8.8,
-                       },
-                     ],
-                     record_batch.collect(&:to_h))
-      end
-    end
-  end
-
-  sub_test_case("#each") do
-    test("without block") do
-      buffer = Arrow::ResizableBuffer.new(1024)
-      Arrow::Table.new(number: [1, 2, 3]).save(buffer, format: :arrows)
-      Arrow::BufferInputStream.open(buffer) do |input|
-        reader = Arrow::RecordBatchStreamReader.new(input)
-        assert_equal([
-                       Arrow::RecordBatch.new(number: [1, 2, 3]),
-                     ],
-                     reader.each.to_a)
-      end
-    end
-  end
-end