red-arrow 13.0.0 → 14.0.1

data/test/each-raw-record/test-struct-array.rb

@@ -0,0 +1,529 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module EachRawRecordStructArrayTests
+  def build_schema(type)
+    field_description = {
+      name: :field,
+    }
+    if type.is_a?(Hash)
+      field_description = field_description.merge(type)
+    else
+      field_description[:type] = type
+    end
+    {
+      column: {
+        type: :struct,
+        fields: [
+          field_description,
+        ],
+      },
+    }
+  end
+
+  def test_null
+    records = [
+      [{"field" => nil}],
+      [nil],
+    ]
+    target = build(:null, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_boolean
+    records = [
+      [{"field" => true}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:boolean, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_int8
+    records = [
+      [{"field" => -(2 ** 7)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:int8, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_uint8
+    records = [
+      [{"field" => (2 ** 8) - 1}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:uint8, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_int16
+    records = [
+      [{"field" => -(2 ** 15)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:int16, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_uint16
+    records = [
+      [{"field" => (2 ** 16) - 1}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:uint16, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_int32
+    records = [
+      [{"field" => -(2 ** 31)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:int32, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_uint32
+    records = [
+      [{"field" => (2 ** 32) - 1}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:uint32, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_int64
+    records = [
+      [{"field" => -(2 ** 63)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:int64, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_uint64
+    records = [
+      [{"field" => (2 ** 64) - 1}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:uint64, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_float
+    records = [
+      [{"field" => -1.0}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:float, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_double
+    records = [
+      [{"field" => -1.0}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:double, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_binary
+    records = [
+      [{"field" => "\xff".b}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:binary, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_string
+    records = [
+      [{"field" => "Ruby"}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:string, records)    
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_date32
+    records = [
+      [{"field" => Date.new(1960, 1, 1)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:date32, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_date64
+    records = [
+      [{"field" => DateTime.new(1960, 1, 1, 2, 9, 30)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:date64, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_timestamp_second
+    records = [
+      [{"field" => Time.parse("1960-01-01T02:09:30Z")}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :timestamp,
+                     unit: :second,
+                   },
+                   records)    
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_timestamp_milli
+    records = [
+      [{"field" => Time.parse("1960-01-01T02:09:30.123Z")}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :timestamp,
+                     unit: :milli,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_timestamp_micro
+    records = [
+      [{"field" => Time.parse("1960-01-01T02:09:30.123456Z")}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :timestamp,
+                     unit: :micro,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_timestamp_nano
+    records = [
+      [{"field" => Time.parse("1960-01-01T02:09:30.123456789Z")}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :timestamp,
+                     unit: :nano,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_time32_second
+    unit = Arrow::TimeUnit::SECOND
+    records = [
+      # 00:10:00
+      [{"field" => Arrow::Time.new(unit, 60 * 10)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :time32,
+                     unit: :second,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_time32_milli
+    unit = Arrow::TimeUnit::MILLI
+    records = [
+      # 00:10:00.123
+      [{"field" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :time32,
+                     unit: :milli,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_time64_micro
+    unit = Arrow::TimeUnit::MICRO
+    records = [
+      # 00:10:00.123456
+      [{"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :time64,
+                     unit: :micro,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_time64_nano
+    unit = Arrow::TimeUnit::NANO
+    records = [
+      # 00:10:00.123456789
+      [{"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :time64,
+                     unit: :nano,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_decimal128
+    records = [
+      [{"field" => BigDecimal("92.92")}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :decimal128,
+                     precision: 8,
+                     scale: 2,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_decimal256
+    records = [
+      [{"field" => BigDecimal("92.92")}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :decimal256,
+                     precision: 38,
+                     scale: 2,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_month_interval
+    records = [
+      [{"field" => 1}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{"field" => {day: 1, millisecond: 100}}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{"field" => {month: 1, day: 1, nanosecond: 100}}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_list
+    records = [
+      [{"field" => [true, nil, false]}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :list,
+                     field: {
+                       name: :sub_element,
+                       type: :boolean,
+                     },
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_struct
+    records = [
+      [{"field" => {"sub_field" => true}}],
+      [nil],
+      [{"field" => nil}],
+      [{"field" => {"sub_field" => nil}}],
+    ]
+    target = build({
+                     type: :struct,
+                     fields: [
+                       {
+                         name: :sub_field,
+                         type: :boolean,
+                       },
+                     ],
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_map
+    records = [
+      [{"field" => {"key1" => true, "key2" => nil}}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build({
+                     type: :map,
+                     key: :string,
+                     item: :boolean,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def remove_union_field_names(records)
+    records.collect do |record|
+      record.collect do |column|
+        if column.nil?
+          column
+        else
+          value = column["field"]
+          value = value.values[0] unless value.nil?
+          {"field" => value}
+        end
+      end
+    end
+  end
+
+  def test_sparse_union
+    records = [
+      [{"field" => {"field1" => true}}],
+      [nil],
+      [{"field" => nil}],
+      [{"field" => {"field2" => 29}}],
+      [{"field" => {"field2" => nil}}],
+    ]
+    target = build({
+                     type: :sparse_union,
+                     fields: [
+                       {
+                         name: :field1,
+                         type: :boolean,
+                       },
+                       {
+                         name: :field2,
+                         type: :uint8,
+                       },
+                     ],
+                     type_codes: [0, 1],
+                   },
+                   records)
+    assert_equal(remove_union_field_names(records),
+                 target.each_raw_record.to_a)
+  end
+
+  def test_dense_union
+    records = [
+      [{"field" => {"field1" => true}}],
+      [nil],
+      [{"field" => nil}],
+      [{"field" => {"field2" => 29}}],
+      [{"field" => {"field2" => nil}}],
+    ]
+    target = build({
+                     type: :dense_union,
+                     fields: [
+                       {
+                         name: :field1,
+                         type: :boolean,
+                       },
+                       {
+                         name: :field2,
+                         type: :uint8,
+                       },
+                     ],
+                     type_codes: [0, 1],
+                   },
+                   records)
+    assert_equal(remove_union_field_names(records),
+                 target.each_raw_record.to_a)
+  end
+
+  def test_dictionary
+    records = [
+      [{"field" => "Ruby"}],
+      [nil],
+      [{"field" => nil}],
+      [{"field" => "GLib"}],
+    ]
+    target = build({
+                     type: :dictionary,
+                     index_data_type: :int8,
+                     value_data_type: :string,
+                     ordered: false,
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+end
+
+class EachRawRecordRecordBatchStructArrayTest < Test::Unit::TestCase
+  include EachRawRecordStructArrayTests
+
+  def build(type, records)
+    Arrow::RecordBatch.new(build_schema(type), records)
+  end
+end
+
+class EachRawRecordTableStructArrayTest < Test::Unit::TestCase
+  include EachRawRecordStructArrayTests
+
+  def build(type, records)
+    Arrow::Table.new(build_schema(type), records)
+  end
+end

data/test/each-raw-record/test-table.rb

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class EachRawRecordTableTest < Test::Unit::TestCase
+  test("2 arrays") do
+    raw_record_batches = [
+      [
+        [true, nil, "Ruby"],
+        [nil, 0, "GLib"],
+        [false, 2 ** 8 - 1, nil],
+      ],
+      [
+        [nil, 10, "A"],
+        [true, 20, "B"],
+        [false, nil, "C"],
+        [nil, 40, nil],
+      ]
+    ]
+    raw_records = raw_record_batches.inject do |all_records, record_batch|
+      all_records + record_batch
+    end
+    schema = [
+      {name: :column0, type: :boolean},
+      {name: :column1, type: :uint8},
+      {name: :column2, type: :string},
+    ]
+    record_batches = raw_record_batches.collect do |record_batch|
+      Arrow::RecordBatch.new(schema, record_batch)
+    end
+    table = Arrow::Table.new(schema, record_batches)
+    assert_equal(raw_records, table.each_raw_record.to_a)
+  end
+end

data/test/raw-records/test-basic-arrays.rb

@@ -157,7 +157,7 @@ module RawRecordsBasicArraysTests
     assert_equal(records, target.raw_records)
   end
 
-  def test_tring
+  def test_string
     records = [
       ["Ruby"],
       [nil],

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 13.0.0
+  version: 14.0.1
 platform: ruby
 authors:
 - Apache Arrow Developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-28 00:00:00.000000000 Z
+date: 2023-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: extpp
   requirement: !ruby/object:Gem::Requirement
@@ -218,6 +232,15 @@ files:
 - lib/arrow/version.rb
 - lib/arrow/writable.rb
 - red-arrow.gemspec
+- test/each-raw-record/test-basic-arrays.rb
+- test/each-raw-record/test-dense-union-array.rb
+- test/each-raw-record/test-dictionary-array.rb
+- test/each-raw-record/test-list-array.rb
+- test/each-raw-record/test-map-array.rb
+- test/each-raw-record/test-multiple-columns.rb
+- test/each-raw-record/test-sparse-union-array.rb
+- test/each-raw-record/test-struct-array.rb
+- test/each-raw-record/test-table.rb
 - test/fixture/TestOrcFile.test1.orc
 - test/fixture/float-integer.csv
 - test/fixture/integer-float.csv
@@ -323,7 +346,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata:
-  msys2_mingw_dependencies: arrow>=13.0.0
+  msys2_mingw_dependencies: arrow>=14.0.1
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -344,6 +367,15 @@ signing_key:
 specification_version: 4
 summary: Red Arrow is the Ruby bindings of Apache Arrow
 test_files:
+- test/each-raw-record/test-basic-arrays.rb
+- test/each-raw-record/test-dense-union-array.rb
+- test/each-raw-record/test-dictionary-array.rb
+- test/each-raw-record/test-list-array.rb
+- test/each-raw-record/test-map-array.rb
+- test/each-raw-record/test-multiple-columns.rb
+- test/each-raw-record/test-sparse-union-array.rb
+- test/each-raw-record/test-struct-array.rb
+- test/each-raw-record/test-table.rb
 - test/fixture/TestOrcFile.test1.orc
 - test/fixture/float-integer.csv
 - test/fixture/integer-float.csv