red-arrow 4.0.1 → 7.0.0

data/lib/arrow/s3-global-options.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class S3GlobalOptions
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          options = new
+          value.each do |k, v|
+            setter = :"#{k}="
+            return unless options.respond_to?(setter)
+            options.__send__(setter, v)
+          end
+          options
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/scalar.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Scalar
+    # @param other [Arrow::Scalar] The scalar to be compared.
+    # @param options [Arrow::EqualOptions, Hash] (nil)
+    #   The options to custom how to compare.
+    #
+    # @return [Boolean]
+    #   `true` if both of them have the same data, `false` otherwise.
+    #
+    # @since 5.0.0
+    def equal_scalar?(other, options=nil)
+      equal_options(other, options)
+    end
+  end
+end

data/lib/arrow/slicer.rb

@@ -16,9 +16,6 @@
 # under the License.
 
 module Arrow
-  # Experimental
-  #
-  # TODO: Almost codes should be implemented in Apache Arrow C++.
   class Slicer
     def initialize(table)
       @table = table
@@ -43,6 +40,21 @@ module Arrow
       super
     end
 
+    module Helper
+      class << self
+        def ensure_boolean(column)
+          case column.data_type
+          when Arrow::BooleanDataType
+            column.data
+          else
+            options = CastOptions.new
+            options.to_data_type = Arrow::BooleanDataType.new
+            Function.find("cast").execute([column.data], options).value
+          end
+        end
+      end
+    end
+
     class Condition
       def evaluate
         message = "Slicer::Condition must define \#evaluate: #{inspect}"
@@ -69,43 +81,28 @@ module Arrow
       end
 
       def evaluate
-        values1 = @condition1.evaluate.each
-        values2 = @condition2.evaluate.each
-        raw_array = []
-        begin
-          loop do
-            value1 = values1.next
-            value2 = values2.next
-            if value1.nil? or value2.nil?
-              raw_array << nil
-            else
-              raw_array << evaluate_value(value1, value2)
-            end
-          end
-        rescue StopIteration
-        end
-        BooleanArray.new(raw_array)
+        function.execute([@condition1.evaluate, @condition2.evaluate]).value
       end
     end
 
     class AndCondition < LogicalCondition
       private
-      def evaluate_value(value1, value2)
-        value1 and value2
+      def function
+        Function.find("and")
       end
     end
 
     class OrCondition < LogicalCondition
       private
-      def evaluate_value(value1, value2)
-        value1 or value2
+      def function
+        Function.find("or")
       end
     end
 
     class XorCondition < LogicalCondition
       private
-      def evaluate_value(value1, value2)
-        value1 ^ value2
+      def function
+        Function.find("xor")
       end
     end
 
@@ -115,21 +112,7 @@ module Arrow
       end
 
       def evaluate
-        data = @column.data
-
-        case @column.data_type
-        when BooleanDataType
-          data
-        else
-          if data.n_chunks == 1
-            data.get_chunk(0).cast(BooleanDataType.new, nil)
-          else
-            arrays = data.each_chunk.collect do |chunk|
-              chunk.cast(BooleanDataType.new, nil)
-            end
-            ChunkedArray.new(arrays)
-          end
-        end
+        Helper.ensure_boolean(@column)
       end
 
       def !@
@@ -187,23 +170,8 @@ module Arrow
       end
 
       def evaluate
-        data = @column.data
-        raw_array = []
-        data.each_chunk do |chunk|
-          if chunk.is_a?(BooleanArray)
-            boolean_array = chunk
-          else
-            boolean_array = chunk.cast(BooleanDataType.new, nil)
-          end
-          boolean_array.each do |value|
-            if value.nil?
-              raw_array << value
-            else
-              raw_array << !value
-            end
-          end
-        end
-        BooleanArray.new(raw_array)
+        data = Helper.ensure_boolean(@column)
+        Function.find("invert").execute([data]).value
       end
 
       def !@
@@ -222,19 +190,10 @@ module Arrow
       end
 
       def evaluate
-        case @value
-        when nil
-          raw_array = @column.collect(&:nil?)
-          BooleanArray.new(raw_array)
+        if @value.nil?
+          Function.find("is_null").execute([@column.data]).value
         else
-          raw_array = @column.collect do |value|
-            if value.nil?
-              nil
-            else
-              @value == value
-            end
-          end
-          BooleanArray.new(raw_array)
+          Function.find("equal").execute([@column.data, @value]).value
         end
       end
     end
@@ -250,25 +209,10 @@ module Arrow
       end
 
       def evaluate
-        case @value
-        when nil
-          if @column.n_nulls.zero?
-            raw_array = [true] * @column.n_rows
-          else
-            raw_array = @column.n_rows.times.collect do |i|
-              @column.valid?(i)
-            end
-          end
-          BooleanArray.new(raw_array)
+        if @value.nil?
+          Function.find("is_valid").execute([@column.data]).value
         else
-          raw_array = @column.collect do |value|
-            if value.nil?
-              nil
-            else
-              @value != value
-            end
-          end
-          BooleanArray.new(raw_array)
+          Function.find("not_equal").execute([@column.data, @value]).value
         end
       end
     end
@@ -284,14 +228,7 @@ module Arrow
       end
 
       def evaluate
-        raw_array = @column.collect do |value|
-          if value.nil?
-            nil
-          else
-            @value > value
-          end
-        end
-        BooleanArray.new(raw_array)
+        Function.find("less").execute([@column.data, @value]).value
       end
     end
 
@@ -306,14 +243,7 @@ module Arrow
       end
 
       def evaluate
-        raw_array = @column.collect do |value|
-          if value.nil?
-            nil
-          else
-            @value >= value
-          end
-        end
-        BooleanArray.new(raw_array)
+        Function.find("less_equal").execute([@column.data, @value]).value
       end
     end
 
@@ -328,14 +258,7 @@ module Arrow
       end
 
       def evaluate
-        raw_array = @column.collect do |value|
-          if value.nil?
-            nil
-          else
-            @value < value
-          end
-        end
-        BooleanArray.new(raw_array)
+        Function.find("greater").execute([@column.data, @value]).value
       end
     end
 
@@ -350,14 +273,7 @@ module Arrow
       end
 
       def evaluate
-        raw_array = @column.collect do |value|
-          if value.nil?
-            nil
-          else
-            @value <= value
-          end
-        end
-        BooleanArray.new(raw_array)
+        Function.find("greater_equal").execute([@column.data, @value]).value
       end
     end
 
@@ -372,18 +288,10 @@ module Arrow
       end
 
       def evaluate
-        values_index = {}
-        @values.each do |value|
-          values_index[value] = true
-        end
-        raw_array = @column.collect do |value|
-          if value.nil?
-            nil
-          else
-            values_index.key?(value)
-          end
-        end
-        BooleanArray.new(raw_array)
+        values = @values
+        values = Array.new(values) unless values.is_a?(Array)
+        options = SetLookupOptions.new(values)
+        Function.find("is_in").execute([@column.data], options).value
       end
     end
 
@@ -398,18 +306,11 @@ module Arrow
       end
 
       def evaluate
-        values_index = {}
-        @values.each do |value|
-          values_index[value] = true
-        end
-        raw_array = @column.collect do |value|
-          if value.nil?
-            nil
-          else
-            not values_index.key?(value)
-          end
-        end
-        BooleanArray.new(raw_array)
+        values = @values
+        values = Array.new(values) unless values.is_a?(Array)
+        options = SetLookupOptions.new(values)
+        booleans = Function.find("is_in").execute([@column.data], options).value
+        Function.find("invert").execute([booleans]).value
       end
     end
 

data/lib/arrow/sort-key.rb

@@ -29,25 +29,26 @@ module Arrow
       #
       #   @return [Arrow::SortKey] The given sort key itself.
       #
-      # @overload resolve(name)
+      # @overload resolve(target)
       #
-      #   Creates a new suitable sort key from column name with
-      #   leading order mark. See {#initialize} for details about
+      #   Creates a new suitable sort key from column name or dot path
+      #   with leading order mark. See {#initialize} for details about
       #   order mark.
       #
       #   @return [Arrow::SortKey] A new suitable sort key.
       #
-      # @overload resolve(name, order)
+      # @overload resolve(target, order)
       #
-      #   Creates a new suitable sort key from column name without
-      #   leading order mark and order. See {#initialize} for details.
+      #   Creates a new suitable sort key from column name or dot path
+      #   without leading order mark and order. See {#initialize} for
+      #   details.
       #
       #   @return [Arrow::SortKey] A new suitable sort key.
       #
       # @since 4.0.0
-      def resolve(name, order=nil)
-        return name if name.is_a?(self)
-        new(name, order)
+      def resolve(target, order=nil)
+        return target if target.is_a?(self)
+        new(target, order)
       end
 
       # @api private
@@ -65,47 +66,49 @@ module Arrow
     private :initialize_raw
     # Creates a new {Arrow::SortKey}.
     #
-    # @overload initialize(name)
+    # @overload initialize(target)
     #
-    #   @param name [Symbol, String] The name of the sort column.
+    #   @param target [Symbol, String] The name or dot path of the
+    #     sort column.
     #
-    #     If `name` is a String, the first character may be processed
-    #     as the "leading order mark". If the first character is `"+"`
-    #     or `"-"`, they are processed as a leading order mark. If the
-    #     first character is processed as a leading order mark, the
-    #     first character is removed from sort column name and
-    #     corresponding order is used. `"+"` uses ascending order and
-    #     `"-"` uses ascending order.
+    #     If `target` is a String, the first character may be
+    #     processed as the "leading order mark". If the first
+    #     character is `"+"` or `"-"`, they are processed as a leading
+    #     order mark. If the first character is processed as a leading
+    #     order mark, the first character is removed from sort column
+    #     target and corresponding order is used. `"+"` uses ascending
+    #     order and `"-"` uses ascending order.
     #
-    #     If `name` is not a String nor `name` doesn't start with the
-    #     leading order mark, sort column name is `name` as-is and
+    #     If `target` is not a String nor `target` doesn't start with the
+    #     leading order mark, sort column target is `target` as-is and
     #     ascending order is used.
     #
     #   @example String without the leading order mark
     #     key = Arrow::SortKey.new("count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
     #   @example String with the "+" leading order mark
     #     key = Arrow::SortKey.new("+count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
     #   @example String with the "-" leading order mark
     #     key = Arrow::SortKey.new("-count")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Symbol that starts with "-"
     #     key = Arrow::SortKey.new(:"-count")
-    #     key.name  # => "-count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "-count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
-    # @overload initialize(name, order)
+    # @overload initialize(target, order)
     #
-    #   @param name [Symbol, String] The name of the sort column.
+    #   @param target [Symbol, String] The name or dot path of the
+    #     sort column.
     #
-    #     No leading order mark processing. The given `name` is used
+    #     No leading order mark processing. The given `target` is used
     #     as-is.
     #
     #   @param order [Symbol, String, Arrow::SortOrder] How to order
@@ -117,29 +120,29 @@ module Arrow
     #
     #   @example No leading order mark processing
     #     key = Arrow::SortKey.new("-count", :ascending)
-    #     key.name  # => "-count"
-    #     key.order # => Arrow::SortOrder::ASCENDING
+    #     key.target # => "-count"
+    #     key.order  # => Arrow::SortOrder::ASCENDING
     #
-    #   @example Order by abbreviated name with Symbol
+    #   @example Order by abbreviated target with Symbol
     #     key = Arrow::SortKey.new("count", :desc)
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Order by String
     #     key = Arrow::SortKey.new("count", "descending")
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     #   @example Order by Arrow::SortOrder
     #     key = Arrow::SortKey.new("count", Arrow::SortOrder::DESCENDING)
-    #     key.name  # => "count"
-    #     key.order # => Arrow::SortOrder::DESCENDING
+    #     key.target # => "count"
+    #     key.order  # => Arrow::SortOrder::DESCENDING
     #
     # @since 4.0.0
-    def initialize(name, order=nil)
-      name, order = normalize_name(name, order)
+    def initialize(target, order=nil)
+      target, order = normalize_target(target, order)
       order = normalize_order(order) || :ascending
-      initialize_raw(name, order)
+      initialize_raw(target, order)
     end
 
     # @return [String] The string representation of this sort key. You
@@ -154,28 +157,31 @@ module Arrow
     # @since 4.0.0
     def to_s
       if order == SortOrder::ASCENDING
-        "+#{name}"
+        "+#{target}"
       else
-        "-#{name}"
+        "-#{target}"
       end
     end
 
+    # For backward compatibility
+    alias_method :name, :target
+
     private
-    def normalize_name(name, order)
-      case name
+    def normalize_target(target, order)
+      case target
       when Symbol
-        return name.to_s, order
+        return target.to_s, order
       when String
-        return name, order if order
-        if name.start_with?("-")
-          return name[1..-1], order || :descending
-        elsif name.start_with?("+")
-          return name[1..-1], order || :ascending
+        return target, order if order
+        if target.start_with?("-")
+          return target[1..-1], order || :descending
+        elsif target.start_with?("+")
+          return target[1..-1], order || :ascending
         else
-          return name, order
+          return target, order
         end
       else
-        return name, order
+        return target, order
       end
     end
 

data/lib/arrow/sort-options.rb

@@ -78,20 +78,20 @@ module Arrow
     #     options.add_sort_key(Arrow::SortKey.new(:price, :descending))
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
-    # @overload add_sort_key(name)
+    # @overload add_sort_key(target)
     #
-    #   @param name [Symbol, String] The sort key name to be
-    #     added. See also {Arrow::SortKey#initialize} for the leading
-    #     order mark for String name.
+    #   @param target [Symbol, String] The sort key name or dot path
+    #     to be added. See also {Arrow::SortKey#initialize} for the
+    #     leading order mark for `String` target.
     #
     #   @example Add a key to sort by "price" column in descending order
     #     options = Arrow::SortOptions.new
     #     options.add_sort_key("-price")
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
-    # @overload add_sort_key(name, order)
+    # @overload add_sort_key(target, order)
     #
-    #   @param name [Symbol, String] The sort key name.
+    #   @param target [Symbol, String] The sort key name or dot path.
     #
     #   @param order [Symbol, String, Arrow::SortOrder] The sort
     #     order. See {Arrow::SortKey#initialize} for details.
@@ -102,8 +102,8 @@ module Arrow
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
     # @since 4.0.0
-    def add_sort_key(name, order=nil)
-      add_sort_key_raw(SortKey.resolve(name, order))
+    def add_sort_key(target, order=nil)
+      add_sort_key_raw(SortKey.resolve(target, order))
     end
   end
 end

data/lib/arrow/source-node-options.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class SourceNodeOptions
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when RecordBatchReader, RecordBatch, Table
+          new(value)
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/string-dictionary-array-builder.rb

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class StringDictionaryArrayBuilder
+    include SymbolValuesAppendable
+
+    private
+    def create_values_array_builder
+      StringArrayBuilder.new
+    end
+  end
+end

data/lib/arrow/symbol-values-appendable.rb

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module SymbolValuesAppendable
+    def append_values(values, is_valids=nil)
+      builder = create_values_array_builder
+      values = values.collect do |value|
+        case value
+        when Symbol
+          value.to_s
+        else
+          value
+        end
+      end
+      builder.append_values(values, is_valids)
+      append_array(builder.finish)
+    end
+  end
+end