red-arrow 6.0.1 → 9.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 853f00543077ecc6ecbc5dcca0f535eda624ece16e2f74bb5ec966f435d60137
-  data.tar.gz: 8e6ea3a343fb7977729a004f61789c5f579b0d2a3c0823faedc4c10425eaec56
+  metadata.gz: 23d181c0c2d49f1479ddae27c02427d75b72f66a31c097e1f0ead363422170ed
+  data.tar.gz: e2c5e259b177bb136d7f492e46ec3411b5f05d190618cec6736494315fc64efb
 SHA512:
-  metadata.gz: cb6de71e011620648821b4b13081386f5ad5a7e77ca7be26554c956a2f1d97ac488ad7f14bf289656fa15504600243fd4e3ba16068d355d6e1559a02a8e1c56f
-  data.tar.gz: 5e1ec7c8e1d306d5c72c282447a23c0b58674ee64f74c6d230834ba495265560a22953e489009bcecff9baa8c49816e5c8e431a1c1c5557cd2bac39a3696df02
+  metadata.gz: a4d7673b6457b4f288d9fc1e0bcc8b9adad239f21dbe722cbb412e81e51d117e8661027731bd4918be19c9ed6c6978f21613ebdd31fbe21271d2175f3d416655
+  data.tar.gz: f300245a89afeb3c8d5e56db26fe866ae666a429429c26155591b430bc47f9d7abe987e35fd0507133b9e5011f78381b6e46797b1072556786fb8fb54e19e9fe

data/Gemfile

@@ -20,3 +20,13 @@
 source "https://rubygems.org/"
 
 gemspec
+
+gem "benchmark-driver"
+gem "bundler"
+gem "faker"
+gem "fiddle", ">= 1.0.9"
+gem "rake"
+gem "redcarpet"
+gem "test-unit"
+gem "webrick"
+gem "yard"

data/ext/arrow/arrow.cpp

@@ -36,6 +36,13 @@ namespace red_arrow {
   ID id_jd;
   ID id_new;
   ID id_to_datetime;
+
+  namespace symbols {
+    VALUE day;
+    VALUE millisecond;
+    VALUE month;
+    VALUE nanosecond;
+  }
 }
 
 extern "C" void Init_arrow() {
@@ -81,4 +88,9 @@ extern "C" void Init_arrow() {
   red_arrow::id_to_datetime = rb_intern("to_datetime");
 
   red_arrow::memory_view::init(mArrow);
+
+  red_arrow::symbols::day = ID2SYM(rb_intern("day"));
+  red_arrow::symbols::millisecond = ID2SYM(rb_intern("millisecond"));
+  red_arrow::symbols::month = ID2SYM(rb_intern("month"));
+  red_arrow::symbols::nanosecond = ID2SYM(rb_intern("nanosecond"));
 }

data/ext/arrow/converters.hpp

@@ -202,6 +202,40 @@ namespace red_arrow {
     //                      const int64_t i) {
     // };
 
+    inline VALUE convert(const arrow::MonthIntervalArray& array,
+                         const int64_t i) {
+      return INT2NUM(array.Value(i));
+    }
+
+    inline VALUE convert(const arrow::DayTimeIntervalArray& array,
+                         const int64_t i) {
+      auto value = rb_hash_new();
+      auto arrow_value = array.Value(i);
+      rb_hash_aset(value,
+                   red_arrow::symbols::day,
+                   INT2NUM(arrow_value.days));
+      rb_hash_aset(value,
+                   red_arrow::symbols::millisecond,
+                   INT2NUM(arrow_value.milliseconds));
+      return value;
+    }
+
+    inline VALUE convert(const arrow::MonthDayNanoIntervalArray& array,
+                         const int64_t i) {
+      auto value = rb_hash_new();
+      auto arrow_value = array.Value(i);
+      rb_hash_aset(value,
+                   red_arrow::symbols::month,
+                   INT2NUM(arrow_value.months));
+      rb_hash_aset(value,
+                   red_arrow::symbols::day,
+                   INT2NUM(arrow_value.days));
+      rb_hash_aset(value,
+                   red_arrow::symbols::nanosecond,
+                   INT2NUM(arrow_value.nanoseconds));
+      return value;
+    }
+
     VALUE convert(const arrow::ListArray& array,
                   const int64_t i);
 
@@ -298,8 +332,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)
@@ -404,8 +439,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)
@@ -506,8 +542,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)
@@ -609,8 +646,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)
@@ -735,8 +773,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)

data/ext/arrow/extconf.rb

@@ -28,7 +28,7 @@ end
 checking_for(checking_message("Homebrew")) do
   platform = NativePackageInstaller::Platform.detect
   if platform.is_a?(NativePackageInstaller::Platform::Homebrew)
-    openssl_prefix = `brew --prefix openssl@1.1`.chomp
+    openssl_prefix = `brew --prefix openssl`.chomp
     unless openssl_prefix.empty?
       PKGConfig.add_path("#{openssl_prefix}/lib/pkgconfig")
     end
@@ -45,9 +45,10 @@ unless required_pkg_config_package([
                                      Arrow::Version::MICRO,
                                    ],
                                    debian: "libarrow-dev",
-                                   redhat: "arrow-devel",
+                                   fedora: "libarrow-devel",
                                    homebrew: "apache-arrow",
-                                   msys2: "arrow")
+                                   msys2: "arrow",
+                                   redhat: "arrow-devel")
   exit(false)
 end
 
@@ -58,9 +59,10 @@ unless required_pkg_config_package([
                                      Arrow::Version::MICRO,
                                    ],
                                    debian: "libarrow-glib-dev",
-                                   redhat: "arrow-glib-devel",
+                                   fedora: "libarrow-glib-devel",
                                    homebrew: "apache-arrow-glib",
-                                   msys2: "arrow")
+                                   msys2: "arrow",
+                                   redhat: "arrow-glib-devel")
   exit(false)
 end
 

data/ext/arrow/raw-records.cpp

@@ -96,8 +96,9 @@ namespace red_arrow {
       VISIT(Time32)
       VISIT(Time64)
       VISIT(Timestamp)
-      // TODO
-      // VISIT(Interval)
+      VISIT(MonthInterval)
+      VISIT(DayTimeInterval)
+      VISIT(MonthDayNanoInterval)
       VISIT(List)
       VISIT(Struct)
       VISIT(Map)

data/ext/arrow/red-arrow.hpp

@@ -47,6 +47,13 @@ namespace red_arrow {
   extern ID id_new;
   extern ID id_to_datetime;
 
+  namespace symbols {
+    extern VALUE day;
+    extern VALUE millisecond;
+    extern VALUE month;
+    extern VALUE nanosecond;
+  }
+
   VALUE array_values(VALUE obj);
   VALUE chunked_array_values(VALUE obj);
 

data/ext/arrow/values.cpp

@@ -77,8 +77,9 @@ namespace red_arrow {
       VISIT(Time32)
       VISIT(Time64)
       VISIT(Timestamp)
-      // TODO
-      // VISIT(Interval)
+      VISIT(MonthInterval)
+      VISIT(DayTimeInterval)
+      VISIT(MonthDayNanoInterval)
       VISIT(List)
       VISIT(Struct)
       VISIT(Map)

data/lib/arrow/array-builder.rb

@@ -33,6 +33,11 @@ module Arrow
         end
         if builder_info
           builder = builder_info[:builder]
+          if builder.nil? and builder_info[:builder_type]
+            builder = create_builder(builder_info)
+          end
+        end
+        if builder
           builder.build(values)
         else
           Arrow::StringArray.new(values)
@@ -121,15 +126,28 @@ module Arrow
             detected: true,
           }
         when BigDecimal
-          if value.to_arrow.is_a?(Decimal128)
-            {
-              builder: Decimal128ArrayBuilder.new,
-            }
-          else
+          builder_info ||= {}
+          if builder_info[:builder] or value.nan? or value.infinite?
             {
-              builder: Decimal256ArrayBuilder.new,
+              builder: StringArrayBuilder.new,
               detected: true,
             }
+          else
+            precision = [builder_info[:precision] || 0, value.precision].max
+            scale = [builder_info[:scale] || 0, value.scale].max
+            if precision <= Decimal128DataType::MAX_PRECISION
+              {
+                builder_type: :decimal128,
+                precision: precision,
+                scale: scale,
+              }
+            else
+              {
+                builder_type: :decimal256,
+                precision: precision,
+                scale: scale,
+              }
+            end
           end
         when ::Array
           sub_builder_info = nil
@@ -154,6 +172,22 @@ module Arrow
           }
         end
       end
+
+      def create_builder(builder_info)
+        builder_type = builder_info[:builder_type]
+        case builder_type
+        when :decimal128
+          data_type = Decimal128DataType.new(builder_info[:precision],
+                                             builder_info[:scale])
+          Decimal128ArrayBuilder.new(data_type)
+        when :decimal256
+          data_type = Decimal256DataType.new(builder_info[:precision],
+                                             builder_info[:scale])
+          Decimal256ArrayBuilder.new(data_type)
+        else
+          nil
+        end
+      end
     end
 
     def build(values)

data/lib/arrow/array-computable.rb

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module ArrayComputable
+    def min(options: nil)
+      compute("min", options: options).value
+    end
+
+    def max(options: nil)
+      compute("max", options: options).value
+    end
+
+    def uniq
+      unique.values
+    end
+
+    private
+    def compute(name, options: nil)
+      Function.find(name).execute([self], options).value
+    end
+  end
+end

data/lib/arrow/array.rb

@@ -18,6 +18,8 @@
 module Arrow
   class Array
     include Enumerable
+
+    include ArrayComputable
     include GenericFilterable
     include GenericTakeable
 
@@ -34,6 +36,20 @@ module Arrow
         return nil unless const_defined?(builder_class_name)
         const_get(builder_class_name)
       end
+
+      # @api private
+      def try_convert(value)
+        case value
+        when ::Array
+          begin
+            new(value)
+          rescue ArgumentError
+            nil
+          end
+        else
+          nil
+        end
+      end
     end
 
     # @param i [Integer]

data/lib/arrow/chunked-array.rb

@@ -18,6 +18,8 @@
 module Arrow
   class ChunkedArray
     include Enumerable
+
+    include ArrayComputable
     include GenericFilterable
     include GenericTakeable
 
@@ -87,5 +89,24 @@ module Arrow
         first_chunk.class.new(to_a)
       end
     end
+
+    def count(options: nil)
+      compute("count", options: options).value
+    end
+
+    def sum(options: nil)
+      compute("sum", options: options).value
+    end
+
+    def unique
+      compute("unique")
+    end
+
+    def cast(target_data_type, options: nil)
+      casted_chunks = chunks.collect do |chunk|
+        chunk.cast(target_data_type, options)
+      end
+      self.class.new(casted_chunks)
+    end
   end
 end

data/lib/arrow/column.rb

@@ -72,5 +72,33 @@ module Arrow
         @field == other.field and
         @data == other.data
     end
+
+    def count(options: nil)
+      @data.count(options: options)
+    end
+
+    def sum(options: nil)
+      @data.sum(options: options)
+    end
+
+    def min(options: nil)
+      @data.min(options: options)
+    end
+
+    def max(options: nil)
+      @data.max(options: options)
+    end
+
+    def unique
+      @data.unique
+    end
+
+    def uniq
+      @data.uniq
+    end
+
+    def cast(target_data_type, options: nil)
+      @data.cast(target_data_type, options: options)
+    end
   end
 end

data/lib/arrow/data-type.rb

@@ -110,7 +110,7 @@ module Arrow
               description[key] = value
             end
           end
-          if type.nil?
+          if type.nil? and self == DataType
             message =
               "data type description must have :type value: #{data_type.inspect}"
             raise ArgumentError, message
@@ -152,6 +152,7 @@ module Arrow
 
       private
       def resolve_class(data_type)
+        return self if data_type.nil?
         components = data_type.to_s.split("_").collect(&:capitalize)
         data_type_name = components.join.gsub(/\AUint/, "UInt")
         data_type_class_name = "#{data_type_name}DataType"

data/lib/arrow/datum.rb

@@ -27,6 +27,8 @@ module Arrow
           ArrayDatum.new(value)
         when ChunkedArray
           ChunkedArrayDatum.new(value)
+        when Column
+          ChunkedArrayDatum.new(value.data)
         when Scalar
           ScalarDatum.new(value)
         when ::Array

data/lib/arrow/day-time-interval-array-builder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DayTimeIntervalArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      if value.is_a?(Hash)
+        Arrow::DayMillisecond.new(value[:day], value[:millisecond])
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/decimal128-array-builder.rb

@@ -42,16 +42,26 @@ module Arrow
     end
 
     private
+    def precision
+      @precision ||= value_data_type.precision
+    end
+
+    def scale
+      @scale ||= value_data_type.scale
+    end
+
     def normalize_value(value)
       case value
-      when String
-        Decimal128.new(value)
-      when Float
-        Decimal128.new(value.to_s)
       when BigDecimal
-        Decimal128.new(value.to_s)
+        if value.nan? or value.infinite?
+          message = "can't use #{value} as an Arrow::Decimal128Array value"
+          raise FloatDomainError, message
+        end
+        integer, decimal = value.to_s("f").split(".", 2)
+        decimal = decimal[0, scale].ljust(scale, "0")
+        Decimal128.new("#{integer}.#{decimal}")
       else
-        value
+        Decimal128.try_convert(value) || value
       end
     end
   end

data/lib/arrow/decimal128.rb

@@ -17,6 +17,20 @@
 
 module Arrow
   class Decimal128
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        when Float
+          new(value.to_s)
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :to_s_raw, :to_s
 
     # @overload to_s

data/lib/arrow/decimal256-array-builder.rb

@@ -45,16 +45,26 @@ module Arrow
     end
 
     private
+    def precision
+      @precision ||= value_data_type.precision
+    end
+
+    def scale
+      @scale ||= value_data_type.scale
+    end
+
     def normalize_value(value)
       case value
-      when String
-        Decimal256.new(value)
-      when Float
-        Decimal256.new(value.to_s)
       when BigDecimal
-        Decimal256.new(value.to_s)
+        if value.nan? or value.infinite?
+          message = "can't use #{value} as an Arrow::Decimal256Array value"
+          raise FloatDomainError, message
+        end
+        integer, decimal = value.to_s("f").split(".", 2)
+        decimal = decimal[0, scale].ljust(scale, "0")
+        Decimal256.new("#{integer}.#{decimal}")
       else
-        value
+        Decimal256.try_convert(value) || value
       end
     end
   end

data/lib/arrow/decimal256.rb

@@ -17,6 +17,20 @@
 
 module Arrow
   class Decimal256
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        when Float
+          new(value.to_s)
+        else
+          nil
+        end
+      end
+    end
+
     alias_method :to_s_raw, :to_s
 
     # @overload to_s