red-arrow 6.0.0 → 8.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c553e573f87398da7d492e8339684bb86b73d1fff02b56cff3277bb87d96a4cd
-  data.tar.gz: 9b97be2878369da0bed9ac8621330926b8a95144d833de6798858a210f88c92a
+  metadata.gz: f43d3875f2e48876cd47533124a9fcf9559e263e63e3954c65fedd36b4e59744
+  data.tar.gz: 2548f661536d3288f4ad8bb25d6587a78ee51d810a85a7d533ccdaf595439a12
 SHA512:
-  metadata.gz: 0bc3526645457a551e25621ba3efaa52a3ad7398e8216c34440651beab58886ec8b52804b7afe227f953b727871980a2ac9a9b1153776499b48cd790ae476304
-  data.tar.gz: 9f3032d5a54a9b0b911edd273916ce68f140c50753d13cb88b950166ea771138f432d738a3a694f44bfff985fcf8329457e97712dca10a7f8a8430d3d8cb82f6
+  metadata.gz: caeffbc164f14db7b056263f1ecd449afe2feec3ccbf7b883c147d415199c9ddc6c20398b00c34884248c103fbb7c55aa225719d8b8e75ac7f49337ee89d557e
+  data.tar.gz: ae5a793e06b63bd80e3499f794193f47f72ff7510a558d2d0e61314068d5bd3a1986e5083bda749b00b73a4615901beb320521eeb360576b93fcb9a1e9d9510a

data/Gemfile

@@ -20,3 +20,13 @@
 source "https://rubygems.org/"
 
 gemspec
+
+gem "benchmark-driver"
+gem "bundler"
+gem "faker"
+gem "fiddle", ">= 1.0.9"
+gem "rake"
+gem "redcarpet"
+gem "test-unit"
+gem "webrick"
+gem "yard"

data/ext/arrow/arrow.cpp

@@ -36,6 +36,13 @@ namespace red_arrow {
   ID id_jd;
   ID id_new;
   ID id_to_datetime;
+
+  namespace symbols {
+    VALUE day;
+    VALUE millisecond;
+    VALUE month;
+    VALUE nanosecond;
+  }
 }
 
 extern "C" void Init_arrow() {
@@ -81,4 +88,9 @@ extern "C" void Init_arrow() {
   red_arrow::id_to_datetime = rb_intern("to_datetime");
 
   red_arrow::memory_view::init(mArrow);
+
+  red_arrow::symbols::day = ID2SYM(rb_intern("day"));
+  red_arrow::symbols::millisecond = ID2SYM(rb_intern("millisecond"));
+  red_arrow::symbols::month = ID2SYM(rb_intern("month"));
+  red_arrow::symbols::nanosecond = ID2SYM(rb_intern("nanosecond"));
 }

data/ext/arrow/converters.hpp

@@ -202,6 +202,40 @@ namespace red_arrow {
     //                      const int64_t i) {
     // };
 
+    inline VALUE convert(const arrow::MonthIntervalArray& array,
+                         const int64_t i) {
+      return INT2NUM(array.Value(i));
+    }
+
+    inline VALUE convert(const arrow::DayTimeIntervalArray& array,
+                         const int64_t i) {
+      auto value = rb_hash_new();
+      auto arrow_value = array.Value(i);
+      rb_hash_aset(value,
+                   red_arrow::symbols::day,
+                   INT2NUM(arrow_value.days));
+      rb_hash_aset(value,
+                   red_arrow::symbols::millisecond,
+                   INT2NUM(arrow_value.milliseconds));
+      return value;
+    }
+
+    inline VALUE convert(const arrow::MonthDayNanoIntervalArray& array,
+                         const int64_t i) {
+      auto value = rb_hash_new();
+      auto arrow_value = array.Value(i);
+      rb_hash_aset(value,
+                   red_arrow::symbols::month,
+                   INT2NUM(arrow_value.months));
+      rb_hash_aset(value,
+                   red_arrow::symbols::day,
+                   INT2NUM(arrow_value.days));
+      rb_hash_aset(value,
+                   red_arrow::symbols::nanosecond,
+                   INT2NUM(arrow_value.nanoseconds));
+      return value;
+    }
+
     VALUE convert(const arrow::ListArray& array,
                   const int64_t i);
 
@@ -298,8 +332,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)
@@ -404,8 +439,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)
@@ -506,8 +542,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)
@@ -609,8 +646,9 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)
@@ -735,8 +773,6 @@ namespace red_arrow {
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
-    // TODO
-    // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
     VISIT(Map)

data/ext/arrow/extconf.rb

@@ -28,7 +28,7 @@ end
 checking_for(checking_message("Homebrew")) do
   platform = NativePackageInstaller::Platform.detect
   if platform.is_a?(NativePackageInstaller::Platform::Homebrew)
-    openssl_prefix = `brew --prefix openssl@1.1`.chomp
+    openssl_prefix = `brew --prefix openssl`.chomp
     unless openssl_prefix.empty?
       PKGConfig.add_path("#{openssl_prefix}/lib/pkgconfig")
     end

data/ext/arrow/raw-records.cpp

@@ -96,8 +96,9 @@ namespace red_arrow {
       VISIT(Time32)
       VISIT(Time64)
       VISIT(Timestamp)
-      // TODO
-      // VISIT(Interval)
+      VISIT(MonthInterval)
+      VISIT(DayTimeInterval)
+      VISIT(MonthDayNanoInterval)
       VISIT(List)
       VISIT(Struct)
       VISIT(Map)

data/ext/arrow/red-arrow.hpp

@@ -47,6 +47,13 @@ namespace red_arrow {
   extern ID id_new;
   extern ID id_to_datetime;
 
+  namespace symbols {
+    extern VALUE day;
+    extern VALUE millisecond;
+    extern VALUE month;
+    extern VALUE nanosecond;
+  }
+
   VALUE array_values(VALUE obj);
   VALUE chunked_array_values(VALUE obj);
 

data/ext/arrow/values.cpp

@@ -77,8 +77,9 @@ namespace red_arrow {
       VISIT(Time32)
       VISIT(Time64)
       VISIT(Timestamp)
-      // TODO
-      // VISIT(Interval)
+      VISIT(MonthInterval)
+      VISIT(DayTimeInterval)
+      VISIT(MonthDayNanoInterval)
       VISIT(List)
       VISIT(Struct)
       VISIT(Map)

data/lib/arrow/datum.rb

@@ -27,6 +27,8 @@ module Arrow
           ArrayDatum.new(value)
         when ChunkedArray
           ChunkedArrayDatum.new(value)
+        when Column
+          ChunkedArrayDatum.new(value.data)
         when Scalar
           ScalarDatum.new(value)
         when ::Array

data/lib/arrow/day-time-interval-array-builder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DayTimeIntervalArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      if value.is_a?(Hash)
+        Arrow::DayMillisecond.new(value[:day], value[:millisecond])
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/function.rb

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Function
+    alias_method :execute_raw, :execute
+    def execute(args, options=nil, context=nil)
+      options = resolve_options(options)
+      execute_raw(args, options, context)
+    end
+    alias_method :call, :execute
+
+    private
+    def resolve_options(options)
+      return nil if options.nil?
+      return options if options.is_a?(FunctionOptions)
+
+      arrow_options_class = options_type&.to_class
+      if arrow_options_class
+        if arrow_options_class.respond_to?(:try_convert)
+          arrow_options = arrow_options_class.try_convert(options)
+          return arrow_options if arrow_options
+        end
+        arrow_options = (default_options || arrow_options_class.new)
+      else
+        arrow_options = default_options
+      end
+      return arrow_options if arrow_options.nil?
+
+      options.each do |key, value|
+        setter = :"#{key}="
+        next unless arrow_options.respond_to?(setter)
+        arrow_options.__send__(setter, value)
+      end
+      arrow_options
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -30,6 +30,7 @@ module Arrow
       require_libraries
       require_extension_library
       gc_guard
+      self.class.start_callback_dispatch_thread
     end
 
     def require_libraries
@@ -58,6 +59,7 @@ module Arrow
       require "arrow/date64-array"
       require "arrow/date64-array-builder"
       require "arrow/datum"
+      require "arrow/day-time-interval-array-builder"
       require "arrow/decimal128"
       require "arrow/decimal128-array"
       require "arrow/decimal128-array-builder"
@@ -76,12 +78,14 @@ module Arrow
       require "arrow/file-system"
       require "arrow/fixed-size-binary-array"
       require "arrow/fixed-size-binary-array-builder"
+      require "arrow/function"
       require "arrow/group"
       require "arrow/list-array-builder"
       require "arrow/list-data-type"
       require "arrow/map-array"
       require "arrow/map-array-builder"
       require "arrow/map-data-type"
+      require "arrow/month-day-nano-interval-array-builder"
       require "arrow/null-array"
       require "arrow/null-array-builder"
       require "arrow/path-extension"
@@ -93,6 +97,7 @@ module Arrow
       require "arrow/record-batch-reader"
       require "arrow/record-batch-stream-reader"
       require "arrow/rolling-window"
+      require "arrow/s3-global-options"
       require "arrow/scalar"
       require "arrow/schema"
       require "arrow/slicer"
@@ -212,5 +217,16 @@ module Arrow
         super
       end
     end
+
+    def prepare_function_info_lock_gvl(function_info, klass)
+      super
+      case klass.name
+      when "Arrow::RecordBatchFileReader"
+        case function_info.name
+        when "new"
+          function_info.lock_gvl_default = false
+        end
+      end
+    end
   end
 end

data/lib/arrow/month-day-nano-interval-array-builder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class MonthDayNanoIntervalArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      if value.is_a?(Hash)
+        Arrow::MonthDayNano.new(value[:month], value[:day], value[:nanosecond])
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/s3-global-options.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class S3GlobalOptions
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          options = new
+          value.each do |k, v|
+            setter = :"#{k}="
+            return unless options.respond_to?(setter)
+            options.__send__(setter, v)
+          end
+          options
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/sort-key.rb

@@ -29,25 +29,26 @@ module Arrow
       #
       #   @return [Arrow::SortKey] The given sort key itself.
       #
-      # @overload resolve(name)
+      # @overload resolve(target)
       #
-      #   Creates a new suitable sort key from column name with
-      #   leading order mark. See {#initialize} for details about
+      #   Creates a new suitable sort key from column name or dot path
+      #   with leading order mark. See {#initialize} for details about
       #   order mark.
       #
       #   @return [Arrow::SortKey] A new suitable sort key.
       #
-      # @overload resolve(name, order)
+      # @overload resolve(target, order)
       #
-      #   Creates a new suitable sort key from column name without
-      #   leading order mark and order. See {#initialize} for details.
+      #   Creates a new suitable sort key from column name or dot path
+      #   without leading order mark and order. See {#initialize} for
+      #   details.
       #
       #   @return [Arrow::SortKey] A new suitable sort key.
       #
       # @since 4.0.0
-      def resolve(name, order=nil)
-        return name if name.is_a?(self)
-        new(name, order)
+      def resolve(target, order=nil)
+        return target if target.is_a?(self)
+        new(target, order)
       end
 
       # @api private
@@ -65,47 +66,49 @@ module Arrow
     private :initialize_raw
     # Creates a new {Arrow::SortKey}.
     #
-    # @overload initialize(name)
+    # @overload initialize(target)
     #
-    #   @param name [Symbol, String] The name of the sort column.
+    #   @param target [Symbol, String] The name or dot path of the
+    #     sort column.
     #
-    #     If `name` is a String, the first character may be processed
-    #     as the "leading order mark". If the first character is `"+"`
-    #     or `"-"`, they are processed as a leading order mark. If the
-    #     first character is processed as a leading order mark, the
-    #     first character is removed from sort column name and
-    #     corresponding order is used. `"+"` uses ascending order and
-    #     `"-"` uses ascending order.
+    #     If `target` is a String, the first character may be
+    #     processed as the "leading order mark". If the first
+    #     character is `"+"` or `"-"`, they are processed as a leading
+    #     order mark. If the first character is processed as a leading
+    #     order mark, the first character is removed from sort column
+    #     target and corresponding order is used. `"+"` uses ascending
+    #     order and `"-"` uses ascending order.
     #
-    #     If `name` is not a String nor `name` doesn't start with the
-    #     leading order mark, sort column name is `name` as-is and
+    #     If `target` is not a String nor `target` doesn't start with the
+    #     leading order mark, sort column target is `target` as-is and
     #     ascending order is used.
     #
     #   @example String without the leading order mark
     #     key = Arrow::SortKey.new("count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
     #   @example String with the "+" leading order mark
     #     key = Arrow::SortKey.new("+count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
     #   @example String with the "-" leading order mark
     #     key = Arrow::SortKey.new("-count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Symbol that starts with "-"
     #     key = Arrow::SortKey.new(:"-count")
-    #     key.name  # => "-count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "-count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
-    # @overload initialize(name, order)
+    # @overload initialize(target, order)
     #
-    #   @param name [Symbol, String] The name of the sort column.
+    #   @param target [Symbol, String] The name or dot path of the
+    #     sort column.
     #
-    #     No leading order mark processing. The given `name` is used
+    #     No leading order mark processing. The given `target` is used
     #     as-is.
     #
     #   @param order [Symbol, String, Arrow::SortOrder] How to order
@@ -117,29 +120,29 @@ module Arrow
     #
     #   @example No leading order mark processing
     #     key = Arrow::SortKey.new("-count", :ascending)
-    #     key.name  # => "-count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "-count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
-    #   @example Order by abbreviated name with Symbol
+    #   @example Order by abbreviated target with Symbol
     #     key = Arrow::SortKey.new("count", :desc)
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Order by String
     #     key = Arrow::SortKey.new("count", "descending")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Order by Arrow::SortOrder
     #     key = Arrow::SortKey.new("count", Arrow::SortOrder::DESCENDING)
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     # @since 4.0.0
-    def initialize(name, order=nil)
-      name, order = normalize_name(name, order)
+    def initialize(target, order=nil)
+      target, order = normalize_target(target, order)
       order = normalize_order(order) || :ascending
-      initialize_raw(name, order)
+      initialize_raw(target, order)
     end
 
     # @return [String] The string representation of this sort key. You
@@ -154,28 +157,31 @@ module Arrow
     # @since 4.0.0
     def to_s
       if order == SortOrder::ASCENDING
-        "+#{name}"
+        "+#{target}"
       else
-        "-#{name}"
+        "-#{target}"
       end
     end
 
+    # For backward compatibility
+    alias_method :name, :target
+
     private
-    def normalize_name(name, order)
-      case name
+    def normalize_target(target, order)
+      case target
       when Symbol
-        return name.to_s, order
+        return target.to_s, order
       when String
-        return name, order if order
-        if name.start_with?("-")
-          return name[1..-1], order || :descending
-        elsif name.start_with?("+")
-          return name[1..-1], order || :ascending
+        return target, order if order
+        if target.start_with?("-")
+          return target[1..-1], order || :descending
+        elsif target.start_with?("+")
+          return target[1..-1], order || :ascending
         else
-          return name, order
+          return target, order
         end
       else
-        return name, order
+        return target, order
       end
     end
 

data/lib/arrow/sort-options.rb

@@ -78,20 +78,20 @@ module Arrow
     #     options.add_sort_key(Arrow::SortKey.new(:price, :descending))
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
-    # @overload add_sort_key(name)
+    # @overload add_sort_key(target)
     #
-    #   @param name [Symbol, String] The sort key name to be
-    #     added. See also {Arrow::SortKey#initialize} for the leading
-    #     order mark for String name.
+    #   @param target [Symbol, String] The sort key name or dot path
+    #     to be added. See also {Arrow::SortKey#initialize} for the
+    #     leading order mark for `String` target.
     #
     #   @example Add a key to sort by "price" column in descending order
     #     options = Arrow::SortOptions.new
     #     options.add_sort_key("-price")
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
-    # @overload add_sort_key(name, order)
+    # @overload add_sort_key(target, order)
     #
-    #   @param name [Symbol, String] The sort key name.
+    #   @param target [Symbol, String] The sort key name or dot path.
     #
     #   @param order [Symbol, String, Arrow::SortOrder] The sort
     #     order. See {Arrow::SortKey#initialize} for details.
@@ -102,8 +102,8 @@ module Arrow
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
     # @since 4.0.0
-    def add_sort_key(name, order=nil)
-      add_sort_key_raw(SortKey.resolve(name, order))
+    def add_sort_key(target, order=nil)
+      add_sort_key_raw(SortKey.resolve(target, order))
     end
   end
 end