red-arrow 16.1.0 → 18.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da8b7abd976ec789610864f01108ed11f819c7f4f3fb89ddc2df953457690932
-  data.tar.gz: 332e1e80b4b71b34300c473d4e92d4207124205312b520ab6a0f55ca8fbe7ff4
+  metadata.gz: 16f24185f6f94f5d5a0179be6d20f289eb2b91fef831c12d1bbd48228c5ee7c0
+  data.tar.gz: 2cb5d5b9adfb6db6b21bb05d35e0dbbbbfbf9a21c9df43ea9b8bca51b93ff19b
 SHA512:
-  metadata.gz: de80fc835bb774e3eb7e83039447aac4dfccf6b9c12d83d0b4b7e583357665a186d99f2fd8b8bd79890caf87f71c7bcd83c9b09231f598e1b7394d622c51864a
-  data.tar.gz: f60d05c58897f02dd72770951a482ea79625b44559f8d213b7fabc49e2fef7bc1297985e8dd7f72dfd14cdeff58010bf4ddf688bdca283edc8493e6ecd5a6f0b
+  metadata.gz: 9521cbc2dd710008ef9e7d671aa65a739867a0903e5878fbbfe99ef2dc4aba71575c8b90937b9eac0068a5c6dc2caf784551e6b206c3ca8feaf1ebb400f7af7e
+  data.tar.gz: 78fa402f9021f9a7511ff518f8fc81e2f748b50a80f60fdb8f7d6cc6ec6a1715ad14c26f2f2c08f76ad1dab0e3a26ca5ce16c710c8868198b806524c8e979f48

data/ext/arrow/extconf.rb

@@ -66,6 +66,13 @@ unless required_pkg_config_package([
   exit(false)
 end
 
+# Old re2.pc (e.g. re2.pc on Ubuntu 20.04) may add -std=c++11. It
+# causes a build error because Apache Arrow C++ requires C++17 or
+# later.
+#
+# We can remove this when we drop support for Ubuntu 20.04.
+$CXXFLAGS.gsub!("-std=c++11", "")
+
 [
   ["glib2", "ext/glib2"],
 ].each do |name, relative_source_dir|
@@ -84,7 +91,7 @@ when /darwin/
   symbols_in_external_bundles.each do |symbol|
     $DLDFLAGS << " -Wl,-U,#{symbol}"
   end
-  mmacosx_version_min = "-mmacosx-version-min=10.15"
+  mmacosx_version_min = "-mmacosx-version-min=12.0"
   $CFLAGS << " #{mmacosx_version_min}"
   $CXXFLAGS << " #{mmacosx_version_min}"
 end

data/lib/arrow/field-containable.rb

@@ -29,7 +29,7 @@ module Arrow
         return nil if index < 0 or index >= n_fields
         get_field(index)
       else
-        message = "field name or index must be String, Symbol or Integer"
+        message = +"field name or index must be String, Symbol or Integer"
         message << ": <#{name_or_index.inspect}>"
         raise ArgumentError, message
       end

data/lib/arrow/loader.rb

@@ -116,6 +116,8 @@ module Arrow
       require "arrow/sparse-union-data-type"
       require "arrow/string-dictionary-array-builder"
       require "arrow/string-array-builder"
+      require "arrow/stream-decoder"
+      require "arrow/stream-listener"
       require "arrow/struct-array"
       require "arrow/struct-array-builder"
       require "arrow/struct-data-type"
@@ -168,6 +170,16 @@ module Arrow
       end
     end
 
+    def rubyish_class_name(info)
+      name = info.name
+      case name
+      when "StreamListener"
+        "StreamListenerRaw"
+      else
+        super
+      end
+    end
+
     def load_object_info(info)
       super
 

data/lib/arrow/stream-decoder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class StreamDecoder
+    def consume(data)
+      case data
+      when Buffer
+        consume_buffer(data)
+      else
+        consume_bytes(data)
+      end
+    end
+  end
+end

data/lib/arrow/stream-listener.rb

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class StreamListener < StreamListenerRaw
+    type_register
+
+    def on_eos
+    end
+
+    def on_record_batch_decoded(record_batch, metadata)
+    end
+
+    def on_schema(schema, filtered_schema)
+    end
+
+    private
+    def virtual_do_on_eos
+      on_eos
+      true
+    end
+
+    def virtual_do_on_record_batch_decoded(record_batch, metadata)
+      on_record_batch_decoded(record_batch, metadata)
+      true
+    end
+
+    def virtual_do_on_schema_decoded(schema, filtered_schema)
+      on_schema_decoded(schema, filtered_schema)
+      true
+    end
+  end
+end

data/lib/arrow/table-formatter.rb

@@ -24,7 +24,8 @@ module Arrow
       attr_reader :head_values
       attr_reader :tail_values
       attr_reader :sample_values
-      def initialize(column, head_values, tail_values)
+      def initialize(table_formatter, column, head_values, tail_values)
+        @table_formatter = table_formatter
         @column = column
         @head_values = head_values
         @tail_values = tail_values
@@ -36,6 +37,15 @@ module Arrow
         @data_type ||= @column.data_type
       end
 
+      def formatted_data_type_name
+        @formatted_data_type_name ||= "(#{data_type.name})"
+      end
+
+      def aligned_data_type_name
+        @aligned_data_type_name ||=
+          "%*s" % [aligned_name.size, formatted_data_type_name]
+      end
+
       def name
         @name ||= @column.name
       end
@@ -63,7 +73,7 @@ module Arrow
             formatted_value = format_value(value[field_name], field_value_width)
             "#{formatted_name}: #{formatted_value}"
           end
-          formatted = "{"
+          formatted = +"{"
           formatted << formatted_values.join(", ")
           formatted << "}"
           "%-*s" % [width, formatted]
@@ -90,9 +100,16 @@ module Arrow
       end
 
       def format_aligned_name(name, data_type, sample_values)
+        if @table_formatter.show_column_type?
+          min_width = formatted_data_type_name.size
+        else
+          min_width = 0
+        end
         case data_type
         when TimestampDataType
-          "%*s" % [::Time.now.iso8601.size, name]
+          width = ::Time.now.iso8601.size
+          width = min_width if width < min_width
+          "%*s" % [width, name]
         when IntegerDataType
           have_null = false
           have_negative = false
@@ -118,9 +135,12 @@ module Arrow
           end
           width += 1 if have_negative # Need "-"
           width = [width, FORMATTED_NULL.size].max if have_null
+          width = min_width if width < min_width
           "%*s" % [width, name]
         when FloatDataType, DoubleDataType
-          "%*s" % [FLOAT_N_DIGITS, name]
+          width = FLOAT_N_DIGITS
+          width = min_width if width < min_width
+          "%*s" % [width, name]
         when StructDataType
           field_widths = data_type.fields.collect do |field|
             field_value_width = compute_field_value_width(field, sample_values)
@@ -130,9 +150,11 @@ module Arrow
           if field_widths.size > 0
             width += (", ".size * (field_widths.size - 1))
           end
+          width = min_width if width < min_width
           "%*s" % [width, name]
         else
-          name
+          width = min_width
+          "%*s" % [width, name]
         end
       end
     end
@@ -143,7 +165,7 @@ module Arrow
     end
 
     def format
-      text = ""
+      text = +""
       n_rows = @table.n_rows
       border = @options[:border] || 10
 
@@ -159,7 +181,7 @@ module Arrow
         else
           tail_values = []
         end
-        ColumnFormatter.new(column, head_values, tail_values)
+        ColumnFormatter.new(self, column, head_values, tail_values)
       end
 
       format_header(text, column_formatters)
@@ -186,5 +208,9 @@ module Arrow
 
       text
     end
+
+    def show_column_type?
+      @options.fetch(:show_column_type, true)
+    end
   end
 end

data/lib/arrow/table-list-formatter.rb

@@ -27,9 +27,9 @@ module Arrow
         text << ("=" * 20 + " #{start_offset + nth_row} " + "=" * 20 + "\n")
         row.each_with_index do |column_value, nth_column|
           column_formatter = column_formatters[nth_column]
-          formatted_name = column_formatter.name
-          formatted_value = column_formatter.format_value(column_value)
-          text << "#{formatted_name}: #{formatted_value}\n"
+          text << column_formatter.name
+          text << "(#{column_formatter.data_type.name})" if show_column_type?
+          text << ": #{column_formatter.format_value(column_value)}\n"
         end
       end
     end

data/lib/arrow/table-table-formatter.rb

@@ -26,6 +26,13 @@ module Arrow
         text << "\t"
         text << column_formatter.aligned_name
       end
+      if show_column_type?
+        text << "\n"
+        column_formatters.each do |column_formatter|
+          text << "\t"
+          text << column_formatter.aligned_data_type_name
+        end
+      end
       text << "\n"
     end
 

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "16.1.0"
+  VERSION = "18.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -49,7 +49,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency("bigdecimal", ">= 3.1.0")
   spec.add_runtime_dependency("csv")
   spec.add_runtime_dependency("extpp", ">= 0.1.1")
-  spec.add_runtime_dependency("gio2", ">= 3.5.0")
+  spec.add_runtime_dependency("gio2", ">= 4.2.3")
   spec.add_runtime_dependency("native-package-installer")
   spec.add_runtime_dependency("pkg-config")
 

data/test/test-csv-loader.rb

@@ -27,80 +27,88 @@ class CSVLoaderTest < Test::Unit::TestCase
     test("String: data: with header") do
       data = fixture_path("with-header-float.csv").read
       assert_equal(<<-TABLE, load_csv(data).to_s)
-	name	     score
-0	alice	 10.100000
-1	bob 	 29.200000
-2	chris	 -1.300000
+	  name	     score
+	(utf8)	  (double)
+0	alice 	 10.100000
+1	bob   	 29.200000
+2	chris 	 -1.300000
       TABLE
     end
 
     test("String: data: without header") do
       data = fixture_path("without-header-float.csv").read
       assert_equal(<<-TABLE, load_csv(data).to_s)
-	0	         1
-0	alice	 10.100000
-1	bob	 29.200000
-2	chris	 -1.300000
+	     0	         1
+	(utf8)	  (double)
+0	alice 	 10.100000
+1	bob   	 29.200000
+2	chris 	 -1.300000
       TABLE
     end
 
     test("String: path: with header") do
       path = fixture_path("with-header-float.csv").to_s
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	name	     score
-0	alice	 10.100000
-1	bob 	 29.200000
-2	chris	 -1.300000
+	  name	     score
+	(utf8)	  (double)
+0	alice 	 10.100000
+1	bob   	 29.200000
+2	chris 	 -1.300000
       TABLE
     end
 
     test("String: path: without header") do
       path = fixture_path("without-header-float.csv").to_s
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	0	         1
-0	alice	 10.100000
-1	bob	 29.200000
-2	chris	 -1.300000
+	     0	         1
+	(utf8)	  (double)
+0	alice 	 10.100000
+1	bob   	 29.200000
+2	chris 	 -1.300000
       TABLE
     end
 
     test("Pathname: with header") do
       path = fixture_path("with-header-float.csv")
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	name	     score
-0	alice	 10.100000
-1	bob 	 29.200000
-2	chris	 -1.300000
+	  name	     score
+	(utf8)	  (double)
+0	alice 	 10.100000
+1	bob   	 29.200000
+2	chris 	 -1.300000
       TABLE
     end
 
     test("Pathname: without header") do
       path = fixture_path("without-header-float.csv")
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	0	         1
-0	alice	 10.100000
-1	bob	 29.200000
-2	chris	 -1.300000
+	     0	         1
+	(utf8)	  (double)
+0	alice 	 10.100000
+1	bob   	 29.200000
+2	chris 	 -1.300000
       TABLE
     end
 
     test("null: with double quote") do
       path = fixture_path("null-with-double-quote.csv").to_s
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	name	 score
-0	alice	    10
-1	bob 	(null)
-2	chris	    -1
+	  name	 score
+	(utf8)	(int8)
+0	alice 	    10
+1	bob   	(null)
+2	chris 	    -1
       TABLE
     end
 
     test("null: without double quote") do
       path = fixture_path("null-without-double-quote.csv").to_s
       assert_equal(<<-TABLE, load_csv(path).to_s)
-	name	 score
-0	alice	    10
-1	bob 	(null)
-2	chris	    -1
+	  name	 score
+	(utf8)	(int8)
+0	alice 	    10
+1	bob   	(null)
+2	chris 	    -1
       TABLE
     end
 

data/test/test-decimal128-array.rb

@@ -38,4 +38,10 @@ class Decimal128ArrayTest < Test::Unit::TestCase
                    array.to_a)
     end
   end
+
+  def test_zero
+    array = Arrow::Decimal128Array.new({precision: 38, scale: 9},
+                                       [BigDecimal("0")])
+    assert_equal(BigDecimal("0"), array[0])
+  end
 end

data/test/test-decimal256-array.rb

@@ -38,4 +38,10 @@ class Decimal256ArrayTest < Test::Unit::TestCase
                    array.to_a)
     end
   end
+
+  def test_zero
+    array = Arrow::Decimal256Array.new({precision: 38, scale: 9},
+                                       [BigDecimal("0")])
+    assert_equal(BigDecimal("0"), array[0])
+  end
 end

data/test/test-group.rb

@@ -43,6 +43,7 @@ class GroupTest < Test::Unit::TestCase
       table = Arrow::Table.new(raw_table)
       assert_equal(<<-TABLE, table.group(:time).count.to_s)
 	                     time	count(int)
+	              (timestamp)	   (int64)
 0	#{time_values[0].iso8601}	         1
 1	#{time_values[1].iso8601}	         1
       TABLE
@@ -53,6 +54,7 @@ class GroupTest < Test::Unit::TestCase
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).count.to_s)
 	group_key1	count(group_key2)	count(int)	count(uint)	count(float)	count(string)
+	   (uint8)	          (int64)	   (int64)	    (int64)	     (int64)	      (int64)
 0	         1	                2	         2	          1	           1	            2
 1	         2	                1	         0	          1	           1	            1
 2	         3	                3	         3	          3	           3	            2
@@ -62,6 +64,7 @@ class GroupTest < Test::Unit::TestCase
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).count.to_s)
 	group_key1	group_key2	count(int)	count(uint)	count(float)	count(string)
+	   (uint8)	   (uint8)	   (int64)	    (int64)	     (int64)	      (int64)
 0	         1	         1	         2	          1	           1	            2
 1	         2	         1	         0	          1	           1	            1
 2	         3	         1	         1	          1	           1	            0
@@ -73,6 +76,7 @@ class GroupTest < Test::Unit::TestCase
       group = @table.group(:group_key1, :group_key2)
       assert_equal(<<-TABLE, group.count(:int, :uint).to_s)
 	group_key1	group_key2	count(int)	count(uint)
+	   (uint8)	   (uint8)	   (int64)	    (int64)
 0	         1	         1	         2	          1
 1	         2	         1	         0	          1
 2	         3	         1	         1	          1
@@ -85,6 +89,7 @@ class GroupTest < Test::Unit::TestCase
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).sum.to_s)
 	group_key1	sum(group_key2)	sum(int)	sum(uint)	sum(float)
+	   (uint8)	       (uint64)	 (int64)	 (uint64)	  (double)
 0	         1	              2	      -3	        1	  2.200000
 1	         2	              1	  (null)	        3	  3.300000
 2	         3	              5	     -15	       15	 16.500000
@@ -94,6 +99,7 @@ class GroupTest < Test::Unit::TestCase
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).sum.to_s)
 	group_key1	group_key2	sum(int)	sum(uint)	sum(float)
+	   (uint8)	   (uint8)	 (int64)	 (uint64)	  (double)
 0	         1	         1	      -3	        1	  2.200000
 1	         2	         1	  (null)	        3	  3.300000
 2	         3	         1	      -4	        4	  4.400000
@@ -106,6 +112,7 @@ class GroupTest < Test::Unit::TestCase
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).mean.to_s)
 	group_key1	mean(group_key2)	 mean(int)	mean(uint)	mean(float)
+	   (uint8)	        (double)	  (double)	  (double)	   (double)
 0	         1	        1.000000	 -1.500000	  1.000000	   2.200000
 1	         2	        1.000000	    (null)	  3.000000	   3.300000
 2	         3	        1.666667	 -5.000000	  5.000000	   5.500000
@@ -115,6 +122,7 @@ class GroupTest < Test::Unit::TestCase
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).mean.to_s)
 	group_key1	group_key2	 mean(int)	mean(uint)	mean(float)
+	   (uint8)	   (uint8)	  (double)	  (double)	   (double)
 0	         1	         1	 -1.500000	  1.000000	   2.200000
 1	         2	         1	    (null)	  3.000000	   3.300000
 2	         3	         1	 -4.000000	  4.000000	   4.400000
@@ -127,6 +135,7 @@ class GroupTest < Test::Unit::TestCase
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).min.to_s)
 	group_key1	min(group_key2)	min(int)	min(uint)	min(float)
+	   (uint8)	        (uint8)	 (int32)	 (uint32)	   (float)
 0	         1	              1	      -2	        1	  2.200000
 1	         2	              1	  (null)	        3	  3.300000
 2	         3	              1	      -6	        4	  4.400000
@@ -136,6 +145,7 @@ class GroupTest < Test::Unit::TestCase
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).min.to_s)
 	group_key1	group_key2	min(int)	min(uint)	min(float)
+	   (uint8)	   (uint8)	 (int32)	 (uint32)	   (float)
 0	         1	         1	      -2	        1	  2.200000
 1	         2	         1	  (null)	        3	  3.300000
 2	         3	         1	      -4	        4	  4.400000
@@ -148,6 +158,7 @@ class GroupTest < Test::Unit::TestCase
     test("single") do
       assert_equal(<<-TABLE, @table.group(:group_key1).max.to_s)
 	group_key1	max(group_key2)	max(int)	max(uint)	max(float)
+	   (uint8)	        (uint8)	 (int32)	 (uint32)	   (float)
 0	         1	              1	      -1	        1	  2.200000
 1	         2	              1	  (null)	        3	  3.300000
 2	         3	              2	      -4	        6	  6.600000
@@ -157,6 +168,7 @@ class GroupTest < Test::Unit::TestCase
     test("multiple") do
       assert_equal(<<-TABLE, @table.group(:group_key1, :group_key2).max.to_s)
 	group_key1	group_key2	max(int)	max(uint)	max(float)
+	   (uint8)	   (uint8)	 (int32)	 (uint32)	   (float)
 0	         1	         1	      -1	        1	  2.200000
 1	         2	         1	  (null)	        3	  3.300000
 2	         3	         1	      -4	        4	  4.400000
@@ -170,6 +182,7 @@ class GroupTest < Test::Unit::TestCase
       group = @table.group(:group_key1, :group_key2)
       assert_equal(<<-TABLE, group.aggregate("count(int)", "sum(uint)").to_s)
 	group_key1	group_key2	count(int)	sum(uint)
+	   (uint8)	   (uint8)	   (int64)	 (uint64)
 0	         1	         1	         2	        1
 1	         2	         1	         0	        3
 2	         3	         1	         1	        4

data/test/test-schema.rb

@@ -95,7 +95,7 @@ class SchemaTest < Test::Unit::TestCase
 
       test("[invalid]") do
         invalid = []
-        message = "field name or index must be String, Symbol or Integer"
+        message = +"field name or index must be String, Symbol or Integer"
         message << ": <#{invalid.inspect}>"
         assert_raise(ArgumentError.new(message)) do
           @schema[invalid]