red-arrow 0.17.0 → 3.0.0

data/lib/arrow/struct-array-builder.rb

@@ -32,7 +32,7 @@ module Arrow
       case index_or_name
       when String, Symbol
         name = index_or_name
-        (@name_to_builder ||= build_name_to_builder)[name.to_s]
+        cached_name_to_builder[name.to_s]
       else
         index = index_or_name
         cached_field_builders[index]
@@ -70,13 +70,18 @@ module Arrow
           append_null
         when ::Array
           append_value_raw
-          value.each_with_index do |sub_value, i|
-            self[i].append(sub_value)
+          cached_field_builders.zip(value) do |builder, sub_value|
+            builder.append(sub_value)
           end
         when Hash
           append_value_raw
+          local_name_to_builder = cached_name_to_builder.dup
           value.each do |name, sub_value|
-            self[name].append(sub_value)
+            builder = local_name_to_builder.delete(name.to_s)
+            builder.append(sub_value)
+          end
+          local_name_to_builder.each do |_, builder|
+            builder.append_null
           end
         else
           message =
@@ -108,9 +113,6 @@ module Arrow
     alias_method :append_null_raw, :append_null
     def append_null
       append_null_raw
-      cached_field_builders.each do |builder|
-        builder.append_null
-      end
     end
 
     # @since 0.12.0
@@ -136,5 +138,9 @@ module Arrow
       end
       name_to_builder
     end
+
+    def cached_name_to_builder
+      @name_to_builder ||= build_name_to_builder
+    end
   end
 end

data/lib/arrow/table-saver.rb

@@ -155,13 +155,13 @@ module Arrow
     end
 
     def save_as_feather
+      properties = FeatherWriteProperties.new
+      properties.class.properties.each do |name|
+        value = @options[name.to_sym]
+        next if value.nil?
+        properties.__send__("#{name}=", value)
+      end
       open_raw_output_stream do |output|
-        properties = FeatherWriteProperties.new
-        properties.class.properties.each do |name|
-          value = @options[name.to_sym]
-          next if value.nil?
-          properties.__send__("#{name}=", value)
-        end
         @table.write_as_feather(output, properties)
       end
     end

data/lib/arrow/table.rb

@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require "arrow/raw-table-converter"
+
 module Arrow
   class Table
     include ColumnContainable
@@ -81,14 +83,6 @@ module Arrow
     #     `Array`.
     #
     #   @example Create a table from column name and values
-    #     count_chunks = [
-    #       Arrow::UInt32Array.new([0, 2]),
-    #       Arrow::UInt32Array.new([nil, 4]),
-    #     ]
-    #     visible_chunks = [
-    #       Arrow::BooleanArray.new([true]),
-    #       Arrow::BooleanArray.new([nil, nil, false]),
-    #     ]
     #     Arrow::Table.new("count" => [0, 2, nil, 4],
     #                      "visible" => [true, nil, nil, false])
     #
@@ -169,22 +163,9 @@ module Arrow
       n_args = args.size
       case n_args
       when 1
-        if args[0][0].is_a?(Column)
-          columns = args[0]
-          fields = columns.collect(&:field)
-          values = columns.collect(&:data)
-          schema = Schema.new(fields)
-        else
-          raw_table = args[0]
-          fields = []
-          values = []
-          raw_table.each do |name, array|
-            array = ArrayBuilder.build(array) if array.is_a?(::Array)
-            fields << Field.new(name.to_s, array.value_data_type)
-            values << array
-          end
-          schema = Schema.new(fields)
-        end
+        raw_table_converter = RawTableConverter.new(args[0])
+        schema = raw_table_converter.schema
+        values = raw_table_converter.values
       when 2
         schema = args[0]
         schema = Schema.new(schema) unless schema.is_a?(Schema)

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "0.17.0"
+  VERSION = "3.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -46,6 +46,7 @@ Gem::Specification.new do |spec|
   spec.test_files += Dir.glob("test/**/*")
   spec.extensions = ["ext/arrow/extconf.rb"]
 
+  spec.add_runtime_dependency("bigdecimal", ">= 2.0.3")
   spec.add_runtime_dependency("extpp", ">= 0.0.7")
   spec.add_runtime_dependency("gio2", ">= 3.3.6")
   spec.add_runtime_dependency("native-package-installer")

data/test/raw-records/test-basic-arrays.rb

@@ -329,6 +329,23 @@ module RawRecordsBasicArraysTests
                    records)
     assert_equal(records, target.raw_records)
   end
+
+  def test_decimal256
+    records = [
+      [BigDecimal("92.92")],
+      [nil],
+      [BigDecimal("29.29")],
+    ]
+    target = build({
+                     column: {
+                       type: :decimal256,
+                       precision: 38,
+                       scale: 2,
+                     }
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
 end
 
 class RawRecordsRecordBatchBasicArraysTest < Test::Unit::TestCase

data/test/raw-records/test-dense-union-array.rb

@@ -56,10 +56,7 @@ module RawRecordsDenseUnionArrayTests
     end
     records.each do |record|
       column = record[0]
-      if column.nil?
-        type_ids << nil
-        offsets << 0
-      elsif column.key?("0")
+      if column.key?("0")
         type_id = type_codes[0]
         type_ids << type_id
         offsets << (type_ids.count(type_id) - 1)
@@ -82,7 +79,6 @@ module RawRecordsDenseUnionArrayTests
   def test_null
     records = [
       [{"0" => nil}],
-      [nil],
     ]
     target = build(:null, records)
     assert_equal(records, target.raw_records)
@@ -91,7 +87,6 @@ module RawRecordsDenseUnionArrayTests
   def test_boolean
     records = [
       [{"0" => true}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:boolean, records)
@@ -101,7 +96,6 @@ module RawRecordsDenseUnionArrayTests
   def test_int8
     records = [
       [{"0" => -(2 ** 7)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int8, records)
@@ -111,7 +105,6 @@ module RawRecordsDenseUnionArrayTests
   def test_uint8
     records = [
       [{"0" => (2 ** 8) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint8, records)
@@ -121,7 +114,6 @@ module RawRecordsDenseUnionArrayTests
   def test_int16
     records = [
       [{"0" => -(2 ** 15)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int16, records)
@@ -131,7 +123,6 @@ module RawRecordsDenseUnionArrayTests
   def test_uint16
     records = [
       [{"0" => (2 ** 16) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint16, records)
@@ -141,7 +132,6 @@ module RawRecordsDenseUnionArrayTests
   def test_int32
     records = [
       [{"0" => -(2 ** 31)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int32, records)
@@ -151,7 +141,6 @@ module RawRecordsDenseUnionArrayTests
   def test_uint32
     records = [
       [{"0" => (2 ** 32) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint32, records)
@@ -161,7 +150,6 @@ module RawRecordsDenseUnionArrayTests
   def test_int64
     records = [
       [{"0" => -(2 ** 63)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int64, records)
@@ -171,7 +159,6 @@ module RawRecordsDenseUnionArrayTests
   def test_uint64
     records = [
       [{"0" => (2 ** 64) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint64, records)
@@ -181,7 +168,6 @@ module RawRecordsDenseUnionArrayTests
   def test_float
     records = [
       [{"0" => -1.0}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:float, records)
@@ -191,7 +177,6 @@ module RawRecordsDenseUnionArrayTests
   def test_double
     records = [
       [{"0" => -1.0}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:double, records)
@@ -201,7 +186,6 @@ module RawRecordsDenseUnionArrayTests
   def test_binary
     records = [
       [{"0" => "\xff".b}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:binary, records)
@@ -211,7 +195,6 @@ module RawRecordsDenseUnionArrayTests
   def test_string
     records = [
       [{"0" => "Ruby"}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:string, records)
@@ -221,7 +204,6 @@ module RawRecordsDenseUnionArrayTests
   def test_date32
     records = [
       [{"0" => Date.new(1960, 1, 1)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:date32, records)
@@ -231,7 +213,6 @@ module RawRecordsDenseUnionArrayTests
   def test_date64
     records = [
       [{"0" => DateTime.new(1960, 1, 1, 2, 9, 30)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:date64, records)
@@ -241,7 +222,6 @@ module RawRecordsDenseUnionArrayTests
   def test_timestamp_second
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -255,7 +235,6 @@ module RawRecordsDenseUnionArrayTests
   def test_timestamp_milli
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -269,7 +248,6 @@ module RawRecordsDenseUnionArrayTests
   def test_timestamp_micro
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123456Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -283,7 +261,6 @@ module RawRecordsDenseUnionArrayTests
   def test_timestamp_nano
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123456789Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -299,7 +276,6 @@ module RawRecordsDenseUnionArrayTests
     records = [
       # 00:10:00
       [{"0" => Arrow::Time.new(unit, 60 * 10)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -315,7 +291,6 @@ module RawRecordsDenseUnionArrayTests
     records = [
       # 00:10:00.123
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -331,7 +306,6 @@ module RawRecordsDenseUnionArrayTests
     records = [
       # 00:10:00.123456
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -347,7 +321,6 @@ module RawRecordsDenseUnionArrayTests
     records = [
       # 00:10:00.123456789
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -361,7 +334,6 @@ module RawRecordsDenseUnionArrayTests
   def test_decimal128
     records = [
       [{"0" => BigDecimal("92.92")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -373,10 +345,23 @@ module RawRecordsDenseUnionArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_decimal256
+    records = [
+      [{"0" => BigDecimal("92.92")}],
+      [{"1" => nil}],
+    ]
+    target = build({
+                     type: :decimal256,
+                     precision: 38,
+                     scale: 2,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"0" => [true, nil, false]}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -393,7 +378,6 @@ module RawRecordsDenseUnionArrayTests
   def test_struct
     records = [
       [{"0" => {"sub_field" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"sub_field" => nil}}],
     ]
@@ -414,7 +398,6 @@ module RawRecordsDenseUnionArrayTests
     omit("Need to add support for SparseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"field2" => nil}}],
     ]
@@ -440,7 +423,6 @@ module RawRecordsDenseUnionArrayTests
     omit("Need to add support for DenseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"field2" => nil}}],
     ]
@@ -466,7 +448,6 @@ module RawRecordsDenseUnionArrayTests
     omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [{"0" => "Ruby"}],
-      [nil],
       [{"1" => nil}],
       [{"0" => "GLib"}],
     ]

data/test/raw-records/test-list-array.rb

@@ -379,6 +379,26 @@ module RawRecordsListArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_decimal256
+    records = [
+      [
+        [
+          BigDecimal("92.92"),
+          nil,
+          BigDecimal("29.29"),
+        ],
+      ],
+      [nil],
+    ]
+    target = build({
+                     type: :decimal256,
+                     precision: 38,
+                     scale: 2,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [

data/test/raw-records/test-sparse-union-array.rb

@@ -51,9 +51,7 @@ module RawRecordsSparseUnionArrayTests
     end
     records.each do |record|
       column = record[0]
-      if column.nil?
-        type_ids << nil
-      elsif column.key?("0")
+      if column.key?("0")
         type_ids << type_codes[0]
       elsif column.key?("1")
         type_ids << type_codes[1]
@@ -71,7 +69,6 @@ module RawRecordsSparseUnionArrayTests
   def test_null
     records = [
       [{"0" => nil}],
-      [nil],
     ]
     target = build(:null, records)
     assert_equal(records, target.raw_records)
@@ -80,7 +77,6 @@ module RawRecordsSparseUnionArrayTests
   def test_boolean
     records = [
       [{"0" => true}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:boolean, records)
@@ -90,7 +86,6 @@ module RawRecordsSparseUnionArrayTests
   def test_int8
     records = [
       [{"0" => -(2 ** 7)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int8, records)
@@ -100,7 +95,6 @@ module RawRecordsSparseUnionArrayTests
   def test_uint8
     records = [
       [{"0" => (2 ** 8) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint8, records)
@@ -110,7 +104,6 @@ module RawRecordsSparseUnionArrayTests
   def test_int16
     records = [
       [{"0" => -(2 ** 15)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int16, records)
@@ -120,7 +113,6 @@ module RawRecordsSparseUnionArrayTests
   def test_uint16
     records = [
       [{"0" => (2 ** 16) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint16, records)
@@ -130,7 +122,6 @@ module RawRecordsSparseUnionArrayTests
   def test_int32
     records = [
       [{"0" => -(2 ** 31)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int32, records)
@@ -140,7 +131,6 @@ module RawRecordsSparseUnionArrayTests
   def test_uint32
     records = [
       [{"0" => (2 ** 32) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint32, records)
@@ -150,7 +140,6 @@ module RawRecordsSparseUnionArrayTests
   def test_int64
     records = [
       [{"0" => -(2 ** 63)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:int64, records)
@@ -160,7 +149,6 @@ module RawRecordsSparseUnionArrayTests
   def test_uint64
     records = [
       [{"0" => (2 ** 64) - 1}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:uint64, records)
@@ -170,7 +158,6 @@ module RawRecordsSparseUnionArrayTests
   def test_float
     records = [
       [{"0" => -1.0}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:float, records)
@@ -180,7 +167,6 @@ module RawRecordsSparseUnionArrayTests
   def test_double
     records = [
       [{"0" => -1.0}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:double, records)
@@ -190,7 +176,6 @@ module RawRecordsSparseUnionArrayTests
   def test_binary
     records = [
       [{"0" => "\xff".b}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:binary, records)
@@ -200,7 +185,6 @@ module RawRecordsSparseUnionArrayTests
   def test_string
     records = [
       [{"0" => "Ruby"}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:string, records)
@@ -210,7 +194,6 @@ module RawRecordsSparseUnionArrayTests
   def test_date32
     records = [
       [{"0" => Date.new(1960, 1, 1)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:date32, records)
@@ -220,7 +203,6 @@ module RawRecordsSparseUnionArrayTests
   def test_date64
     records = [
       [{"0" => DateTime.new(1960, 1, 1, 2, 9, 30)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build(:date64, records)
@@ -230,7 +212,6 @@ module RawRecordsSparseUnionArrayTests
   def test_timestamp_second
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -244,7 +225,6 @@ module RawRecordsSparseUnionArrayTests
   def test_timestamp_milli
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -258,7 +238,6 @@ module RawRecordsSparseUnionArrayTests
   def test_timestamp_micro
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123456Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -272,7 +251,6 @@ module RawRecordsSparseUnionArrayTests
   def test_timestamp_nano
     records = [
       [{"0" => Time.parse("1960-01-01T02:09:30.123456789Z")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -288,7 +266,6 @@ module RawRecordsSparseUnionArrayTests
     records = [
       # 00:10:00
       [{"0" => Arrow::Time.new(unit, 60 * 10)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -304,7 +281,6 @@ module RawRecordsSparseUnionArrayTests
     records = [
       # 00:10:00.123
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -320,7 +296,6 @@ module RawRecordsSparseUnionArrayTests
     records = [
       # 00:10:00.123456
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -336,7 +311,6 @@ module RawRecordsSparseUnionArrayTests
     records = [
       # 00:10:00.123456789
       [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -350,7 +324,6 @@ module RawRecordsSparseUnionArrayTests
   def test_decimal128
     records = [
       [{"0" => BigDecimal("92.92")}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -362,10 +335,23 @@ module RawRecordsSparseUnionArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_decimal256
+    records = [
+      [{"0" => BigDecimal("92.92")}],
+      [{"1" => nil}],
+    ]
+    target = build({
+                     type: :decimal256,
+                     precision: 38,
+                     scale: 2,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"0" => [true, nil, false]}],
-      [nil],
       [{"1" => nil}],
     ]
     target = build({
@@ -382,7 +368,6 @@ module RawRecordsSparseUnionArrayTests
   def test_struct
     records = [
       [{"0" => {"sub_field" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"sub_field" => nil}}],
     ]
@@ -403,7 +388,6 @@ module RawRecordsSparseUnionArrayTests
     omit("Need to add support for SparseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"field2" => nil}}],
     ]
@@ -429,7 +413,6 @@ module RawRecordsSparseUnionArrayTests
     omit("Need to add support for DenseUnionArrayBuilder")
     records = [
       [{"0" => {"field1" => true}}],
-      [nil],
       [{"1" => nil}],
       [{"0" => {"field2" => nil}}],
     ]
@@ -455,7 +438,6 @@ module RawRecordsSparseUnionArrayTests
     omit("Need to add support for DictionaryArrayBuilder")
     records = [
       [{"0" => "Ruby"}],
-      [nil],
       [{"1" => nil}],
       [{"0" => "GLib"}],
     ]