red-arrow 13.0.0 → 14.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcbc004b8b61405c6e1143428ca37aac1be3de4d31d73f0a235f9c2e8d7bf5bb
-  data.tar.gz: 508ba001225d07eae018b7552987c6102a8b59c98f4d717b4e88ad1a188d84b3
+  metadata.gz: d76026332781cece25d5056c7b5eff1d650bfb8b794678ee16fe5e733412d39d
+  data.tar.gz: de610617cb966bca7b13b4e8e5d012f1d4ccb9c2e61e1e5db89a7af6f5d46cd4
 SHA512:
-  metadata.gz: 5cbeae509369615824b6c5d41d8f97d4b166ca99d9b1dc3fba4207e726ac22a0147027cb531761ba91245df7d71bb02ab370221ec72359470149ace65b2a01f7
-  data.tar.gz: 2043d58148d1370789ad6b75ed605b7d3a78c1094f2250a09b0857a43d24f37a83a5706b3b96b41dfa7f0dafa22179dde63a4e991e58e85f19002759cfed52b8
+  metadata.gz: 13e0adf265f41336b40cea4e3d96be8a142d920c0a6a15bc0ce343d80408ca7c4220edbe8a4a91a2973fed4ea8229820cd762166e5a9ae4d0418ed3eb1c164c1
+  data.tar.gz: 9cf84a3bd5527a7d19dfaaf3aa3c922543ec6cfccdedb90bd721d68bec1fb4b9851df20e003df302a663bde921c48d15debf2c37dd0e82bde669382e6cf4343d

data/ext/arrow/arrow.cpp

@@ -82,11 +82,17 @@ extern "C" void Init_arrow() {
   rb_define_method(cArrowRecordBatch, "raw_records",
                    reinterpret_cast<rb::RawMethod>(red_arrow::record_batch_raw_records),
                    0);
+  rb_define_method(cArrowRecordBatch, "each_raw_record",
+                   reinterpret_cast<rb::RawMethod>(red_arrow::record_batch_each_raw_record),
+                   0);
 
   auto cArrowTable = rb_const_get_at(mArrow, rb_intern("Table"));
   rb_define_method(cArrowTable, "raw_records",
                    reinterpret_cast<rb::RawMethod>(red_arrow::table_raw_records),
                    0);
+  rb_define_method(cArrowTable, "each_raw_record",
+                   reinterpret_cast<rb::RawMethod>(red_arrow::table_each_raw_record),
+                   0);
 
   red_arrow::cDate = rb_const_get(rb_cObject, rb_intern("Date"));
 

data/ext/arrow/raw-records.cpp

@@ -144,6 +144,128 @@ namespace red_arrow {
       // The number of columns.
       const int n_columns_;
     };
+
+    class RawRecordsProducer : private Converter, public arrow::ArrayVisitor {
+    public:
+      explicit RawRecordsProducer()
+        : Converter(),
+          record_(Qnil),
+          column_index_(0),
+          row_offset_(0) {
+      }
+
+      void produce(const arrow::RecordBatch& record_batch) {
+        rb::protect([&] {
+          const auto n_columns = record_batch.num_columns();
+          const auto n_rows = record_batch.num_rows();
+          for (int64_t i = 0; i < n_rows; ++i) {
+            record_ = rb_ary_new_capa(n_columns);
+            row_offset_ = i;
+            for (int i = 0; i < n_columns; ++i) {
+              const auto array = record_batch.column(i).get();
+              column_index_ = i;
+              check_status(array->Accept(this),
+                           "[record-batch][each-raw-record]");
+            }
+            rb_yield(record_);
+          }
+          return Qnil;
+        });
+      }
+
+      void produce(const arrow::Table& table) {
+        rb::protect([&] {
+          const auto n_columns = table.num_columns();
+          const auto n_rows = table.num_rows();
+          std::vector<int> chunk_indexes(n_columns);
+          std::vector<int64_t> row_offsets(n_columns);
+          for (int64_t i_row = 0; i_row < n_rows; ++i_row) {
+            record_ = rb_ary_new_capa(n_columns);
+            for (int i_column = 0; i_column < n_columns; ++i_column) {
+              column_index_ = i_column;
+              const auto chunked_array = table.column(i_column).get();
+              auto& chunk_index = chunk_indexes[i_column];
+              auto& row_offset = row_offsets[i_column];
+              auto array = chunked_array->chunk(chunk_index).get();
+              while (array->length() == row_offset) {
+                ++chunk_index;
+                row_offset = 0;
+                array = chunked_array->chunk(chunk_index).get();
+              }
+              row_offset_ = row_offset;
+              check_status(array->Accept(this),
+                           "[table][each-raw-record]");
+              ++row_offset;
+            }
+            rb_yield(record_);
+          }
+
+          return Qnil;
+        });
+      }
+
+#define VISIT(TYPE)                                                     \
+      arrow::Status Visit(const arrow::TYPE ## Array& array) override { \
+        convert(array);                                                 \
+        return arrow::Status::OK();                                     \
+      }
+
+      VISIT(Null)
+      VISIT(Boolean)
+      VISIT(Int8)
+      VISIT(Int16)
+      VISIT(Int32)
+      VISIT(Int64)
+      VISIT(UInt8)
+      VISIT(UInt16)
+      VISIT(UInt32)
+      VISIT(UInt64)
+      VISIT(HalfFloat)
+      VISIT(Float)
+      VISIT(Double)
+      VISIT(Binary)
+      VISIT(String)
+      VISIT(FixedSizeBinary)
+      VISIT(Date32)
+      VISIT(Date64)
+      VISIT(Time32)
+      VISIT(Time64)
+      VISIT(Timestamp)
+      VISIT(MonthInterval)
+      VISIT(DayTimeInterval)
+      VISIT(MonthDayNanoInterval)
+      VISIT(List)
+      VISIT(Struct)
+      VISIT(Map)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
+      VISIT(Dictionary)
+      VISIT(Decimal128)
+      VISIT(Decimal256)
+      // TODO
+      // VISIT(Extension)
+
+#undef VISIT
+
+    private:
+        template <typename ArrayType>
+        void convert(const ArrayType& array) {
+          auto value = Qnil;
+          if (!array.IsNull(row_offset_)) {
+            value = convert_value(array, row_offset_);
+          }
+          rb_ary_store(record_, column_index_, value);
+        }
+
+        // Destination for converted record.
+        VALUE record_;
+
+        // The current column index.
+        int column_index_;
+
+        // The current row offset.
+        int64_t row_offset_;
+    };
   }
 
   VALUE
@@ -181,4 +303,36 @@ namespace red_arrow {
 
     return records;
   }
+
+  VALUE
+  record_batch_each_raw_record(VALUE rb_record_batch) {
+    auto garrow_record_batch = GARROW_RECORD_BATCH(RVAL2GOBJ(rb_record_batch));
+    auto record_batch = garrow_record_batch_get_raw(garrow_record_batch).get();
+    RETURN_SIZED_ENUMERATOR(rb_record_batch, 0, nullptr, record_batch->num_rows());
+
+    try {
+      RawRecordsProducer producer;
+      producer.produce(*record_batch);
+    } catch (rb::State& state) {
+      state.jump();
+    }
+
+    return Qnil;
+  }
+
+  VALUE
+  table_each_raw_record(VALUE rb_table) {
+    auto garrow_table = GARROW_TABLE(RVAL2GOBJ(rb_table));
+    auto table = garrow_table_get_raw(garrow_table).get();
+    RETURN_SIZED_ENUMERATOR(rb_table, 0, nullptr, table->num_rows());
+
+    try {
+      RawRecordsProducer producer;
+      producer.produce(*table);
+    } catch (rb::State& state) {
+      state.jump();
+    }
+
+    return Qnil;
+  }
 }

data/ext/arrow/red-arrow.hpp

@@ -59,6 +59,8 @@ namespace red_arrow {
 
   VALUE record_batch_raw_records(VALUE obj);
   VALUE table_raw_records(VALUE obj);
+  VALUE record_batch_each_raw_record(VALUE obj);
+  VALUE table_each_raw_record(VALUE obj);
 
   inline VALUE time_unit_to_scale(const arrow::TimeUnit::type unit) {
     switch (unit) {

data/lib/arrow/table.rb

@@ -127,7 +127,7 @@ module Arrow
     #     You can also specify schema as primitive Ruby objects.
     #     See {Arrow::Schema#initialize} for details.
     #
-    #   @param arrays [::Array<Arrow::RecordBatch>] The data of the table.
+    #   @param record_batches [::Array<Arrow::RecordBatch>] The data of the table.
     #
     #   @example Create a table from schema and record batches
     #     count_field = Arrow::Field.new("count", :uint32)
@@ -145,7 +145,7 @@ module Arrow
     #     You can also specify schema as primitive Ruby objects.
     #     See {Arrow::Schema#initialize} for details.
     #
-    #   @param arrays [::Array<::Array>] The data of the table as primitive
+    #   @param raw_records [::Array<::Array>] The data of the table as primitive
     #     Ruby objects.
     #
     #   @example Create a table from schema and raw records

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "13.0.0"
+  VERSION = "14.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -47,6 +47,7 @@ Gem::Specification.new do |spec|
   spec.extensions = ["ext/arrow/extconf.rb"]
 
   spec.add_runtime_dependency("bigdecimal", ">= 3.1.0")
+  spec.add_runtime_dependency("csv")
   spec.add_runtime_dependency("extpp", ">= 0.1.1")
   spec.add_runtime_dependency("gio2", ">= 3.5.0")
   spec.add_runtime_dependency("native-package-installer")

data/test/each-raw-record/test-basic-arrays.rb

@@ -0,0 +1,411 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module EachRawRecordBasicArraysTests
+  def test_null
+    records = [
+      [nil],
+      [nil],
+      [nil],
+    ]
+    target = build({column: :null}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_boolean
+    records = [
+      [true],
+      [nil],
+      [false],
+    ]
+    target = build({column: :boolean}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_int8
+    records = [
+      [-(2 ** 7)],
+      [nil],
+      [(2 ** 7) - 1],
+    ]
+    target = build({column: :int8}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_uint8
+    records = [
+      [0],
+      [nil],
+      [(2 ** 8) - 1],
+    ]
+    target = build({column: :uint8}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_int16
+    records = [
+      [-(2 ** 15)],
+      [nil],
+      [(2 ** 15) - 1],
+    ]
+    target = build({column: :int16}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_uint16
+    records = [
+      [0],
+      [nil],
+      [(2 ** 16) - 1],
+    ]
+    target = build({column: :uint16}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_int32
+    records = [
+      [-(2 ** 31)],
+      [nil],
+      [(2 ** 31) - 1],
+    ]
+    target = build({column: :int32}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_uint32
+    records = [
+      [0],
+      [nil],
+      [(2 ** 32) - 1],
+    ]
+    target = build({column: :uint32}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_int64
+    records = [
+      [-(2 ** 63)],
+      [nil],
+      [(2 ** 63) - 1],
+    ]
+    target = build({column: :int64}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_uint64
+    records = [
+      [0],
+      [nil],
+      [(2 ** 64) - 1],
+    ]
+    target = build({column: :uint64}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_half_float
+    records = [
+      [-1.5],
+      [nil],
+      [1.5],
+    ]
+    target = build({column: :half_float}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_float
+    records = [
+      [-1.0],
+      [nil],
+      [1.0],
+    ]
+    target = build({column: :float}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_double
+    records = [
+      [-1.0],
+      [nil],
+      [1.0],
+    ]
+    target = build({column: :double}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_binary
+    records = [
+      ["\x00".b],
+      [nil],
+      ["\xff".b],
+    ]
+    target = build({column: :binary}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_string
+    records = [
+      ["Ruby"],
+      [nil],
+      ["\u3042"], # U+3042 HIRAGANA LETTER A
+    ]
+    target = build({column: :string}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_date32
+    records = [
+      [Date.new(1960, 1, 1)],
+      [nil],
+      [Date.new(2017, 8, 23)],
+    ]
+    target = build({column: :date32}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_date64
+    records = [
+      [DateTime.new(1960, 1, 1, 2, 9, 30)],
+      [nil],
+      [DateTime.new(2017, 8, 23, 14, 57, 2)],
+    ]
+    target = build({column: :date64}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_timestamp_second
+    records = [
+      [Time.parse("1960-01-01T02:09:30Z")],
+      [nil],
+      [Time.parse("2017-08-23T14:57:02Z")],
+    ]
+    target = build({
+                     column: {
+                       type: :timestamp,
+                       unit: :second,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_timestamp_milli
+    records = [
+      [Time.parse("1960-01-01T02:09:30.123Z")],
+      [nil],
+      [Time.parse("2017-08-23T14:57:02.987Z")],
+    ]
+    target = build({
+                     column: {
+                       type: :timestamp,
+                       unit: :milli,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_timestamp_micro
+    records = [
+      [Time.parse("1960-01-01T02:09:30.123456Z")],
+      [nil],
+      [Time.parse("2017-08-23T14:57:02.987654Z")],
+    ]
+    target = build({
+                     column: {
+                       type: :timestamp,
+                       unit: :micro,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_timestamp_nano
+    records = [
+      [Time.parse("1960-01-01T02:09:30.123456789Z")],
+      [nil],
+      [Time.parse("2017-08-23T14:57:02.987654321Z")],
+    ]
+    target = build({
+                     column: {
+                       type: :timestamp,
+                       unit: :nano,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_time32_second
+    unit = Arrow::TimeUnit::SECOND
+    records = [
+      [Arrow::Time.new(unit, 60 * 10)], # 00:10:00
+      [nil],
+      [Arrow::Time.new(unit, 60 * 60 * 2 + 9)], # 02:00:09
+    ]
+    target = build({
+                     column: {
+                       type: :time32,
+                       unit: :second,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_time32_milli
+    unit = Arrow::TimeUnit::MILLI
+    records = [
+      [Arrow::Time.new(unit, (60 * 10) * 1000 + 123)], # 00:10:00.123
+      [nil],
+      [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987)], # 02:00:09.987
+    ]
+    target = build({
+                     column: {
+                       type: :time32,
+                       unit: :milli,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_time64_micro
+    unit = Arrow::TimeUnit::MICRO
+    records = [
+      # 00:10:00.123456
+      [Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)],
+      [nil],
+      # 02:00:09.987654
+      [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654)],
+    ]
+    target = build({
+                     column: {
+                       type: :time64,
+                       unit: :micro,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_time64_nano
+    unit = Arrow::TimeUnit::NANO
+    records = [
+      # 00:10:00.123456789
+      [Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)],
+      [nil],
+      # 02:00:09.987654321
+      [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321)],
+    ]
+    target = build({
+                     column: {
+                       type: :time64,
+                       unit: :nano,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_decimal128
+    records = [
+      [BigDecimal("92.92")],
+      [nil],
+      [BigDecimal("29.29")],
+    ]
+    target = build({
+                     column: {
+                       type: :decimal128,
+                       precision: 8,
+                       scale: 2,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_decimal256
+    records = [
+      [BigDecimal("92.92")],
+      [nil],
+      [BigDecimal("29.29")],
+    ]
+    target = build({
+                     column: {
+                       type: :decimal256,
+                       precision: 38,
+                       scale: 2,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_month_interval
+    records = [
+      [1],
+      [nil],
+      [12],
+    ]
+    target = build({column: :month_interval}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{day: 1, millisecond: 100}],
+      [nil],
+      [{day: 2, millisecond: 300}],
+    ]
+    target = build({column: :day_time_interval}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{month: 1, day: 1, nanosecond: 100}],
+      [nil],
+      [{month: 2, day: 3, nanosecond: 400}],
+    ]
+    target = build({column: :month_day_nano_interval}, records)
+    assert_equal(records, target.each_raw_record.to_a)
+  end
+end
+
+class EachRawRecordRecordBatchBasicArraysTest< Test::Unit::TestCase
+  include EachRawRecordBasicArraysTests
+
+  def build(schema, records)
+    Arrow::RecordBatch.new(schema, records)
+  end
+end
+
+class EachRawRecordTableBasicArraysTest < Test::Unit::TestCase
+  include EachRawRecordBasicArraysTests
+
+  def build(schema, records)
+    record_batch = Arrow::RecordBatch.new(schema, records)
+    # Multiple chunks
+    record_batches = [
+      record_batch.slice(0, 2),
+      record_batch.slice(2, 0), # Empty chunk
+      record_batch.slice(2, record_batch.length - 2),
+    ]
+    Arrow::Table.new(schema, record_batches)
+  end
+end