red-arrow 4.0.0 → 6.0.1

data/lib/arrow/aggregation.rb

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Aggregation
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          function = value[:function]
+          return nil if function.nil?
+          function = function.to_s if function.is_a?(Symbol)
+          return nil unless function.is_a?(String)
+          # TODO: Improve this when we have non hash based aggregate function
+          function = "hash_#{function}" unless function.start_with?("hash_")
+          options = value[:options]
+          input = value[:input]
+          return nil if input.nil?
+          output = value[:output]
+          if output.nil?
+            normalized_function = function.gsub(/\Ahash_/, "")
+            output = "#{normalized_function}(#{input})"
+          end
+          new(function, options, input, output)
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/array-builder.rb

@@ -58,6 +58,11 @@ module Arrow
             builder: StringArrayBuilder.new,
             detected: true,
           }
+        when Symbol
+          {
+            builder: StringDictionaryArrayBuilder.new,
+            detected: true,
+          }
         when Float
           {
             builder: DoubleArrayBuilder.new,

data/lib/arrow/array.rb

@@ -55,6 +55,18 @@ module Arrow
       end
     end
 
+    # @param other [Arrow::Array] The array to be compared.
+    # @param options [Arrow::EqualOptions, Hash] (nil)
+    #   The options to custom how to compare.
+    #
+    # @return [Boolean]
+    #   `true` if both of them have the same data, `false` otherwise.
+    #
+    # @since 5.0.0
+    def equal_array?(other, options=nil)
+      equal_options(other, options)
+    end
+
     def each
       return to_enum(__method__) unless block_given?
 

data/lib/arrow/binary-dictionary-array-builder.rb

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class BinaryDictionaryArrayBuilder
+    include SymbolValuesAppendable
+
+    private
+    def create_values_array_builder
+      BinaryArrayBuilder.new
+    end
+  end
+end

data/lib/arrow/buffer.rb

@@ -17,12 +17,16 @@
 
 module Arrow
   class Buffer
-    alias_method :initialize_raw, :initialize
-    private :initialize_raw
-
-    def initialize(data)
-      @data = data
-      initialize_raw(data)
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        else
+          nil
+        end
+      end
     end
   end
 end

data/lib/arrow/column-containable.rb

@@ -27,6 +27,17 @@ module Arrow
       columns.each(&block)
     end
 
+    # @overload [](name)
+    #   Find a column that has the given name.
+    #
+    #   @param name [String, Symbol] The column name to be found.
+    #   @return [Column] The found column.
+    #
+    # @overload [](index)
+    #   Find the `index`-th column.
+    #
+    #   @param index [Integer] The index to be found.
+    #   @return [Column] The found column.
     def find_column(name_or_index)
       case name_or_index
       when String, Symbol
@@ -40,9 +51,97 @@ module Arrow
         return nil if index < 0 or index >= n_columns
         Column.new(self, index)
       else
-        message = "column name or index must be String, Symbol or Integer"
+        message = "column name or index must be String, Symbol or Integer: "
+        message << name_or_index.inspect
         raise ArgumentError, message
       end
     end
+
+    # Selects columns that are selected by `selectors` and/or `block`
+    # and creates a new container only with the selected columns.
+    #
+    # @param selectors [Array<String, Symbol, Integer, Range>]
+    #   If a selector is `String`, `Symbol` or `Integer`, the selector
+    #   selects a column by {#find_column}.
+    #
+    #   If a selector is `Range`, the selector selects columns by `::Array#[]`.
+    # @yield [column] Gives a column to the block to select columns.
+    #   This uses `::Array#select`.
+    # @yieldparam column [Column] A target column.
+    # @yieldreturn [Boolean] Whether the given column is selected or not.
+    # @return [self.class] The newly created container that only has selected
+    #   columns.
+    def select_columns(*selectors, &block)
+      if selectors.empty?
+        return to_enum(__method__) unless block_given?
+        selected_columns = columns.select(&block)
+      else
+        selected_columns = []
+        selectors.each do |selector|
+          case selector
+          when Range
+            selected_columns.concat(columns[selector])
+          else
+            column = find_column(selector)
+            if column.nil?
+              case selector
+              when String, Symbol
+                message = "unknown column: #{selector.inspect}: #{inspect}"
+                raise KeyError.new(message)
+              else
+                message = "out of index (0..#{n_columns - 1}): "
+                message << "#{selector.inspect}: #{inspect}"
+                raise IndexError.new(message)
+              end
+            end
+            selected_columns << column
+          end
+        end
+        selected_columns = selected_columns.select(&block) if block_given?
+      end
+      self.class.new(selected_columns)
+    end
+
+    # @overload [](name)
+    #   Find a column that has the given name.
+    #
+    #   @param name [String, Symbol] The column name to be found.
+    #   @return [Column] The found column.
+    #   @see #find_column
+    #
+    # @overload [](index)
+    #   Find the `index`-th column.
+    #
+    #   @param index [Integer] The index to be found.
+    #   @return [Column] The found column.
+    #   @see #find_column
+    #
+    # @overload [](range)
+    #   Selects columns that are in `range` and creates a new container
+    #   only with the selected columns.
+    #
+    #   @param range [Range] The range to be selected.
+    #   @return [self.class] The newly created container that only has selected
+    #     columns.
+    #   @see #select_columns
+    #
+    # @overload [](selectors)
+    #   Selects columns that are selected by `selectors` and creates a
+    #   new container only with the selected columns.
+    #
+    #   @param selectors [Array] The selectors that are used to select columns.
+    #   @return [self.class] The newly created container that only has selected
+    #     columns.
+    #   @see #select_columns
+    def [](selector)
+      case selector
+      when ::Array
+        select_columns(*selector)
+      when Range
+        select_columns(selector)
+      else
+        find_column(selector)
+      end
+    end
   end
 end

data/lib/arrow/constructor-arguments-gc-guardable.rb

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module ConstructorArgumentsGCGuardable
+    def initialize(*args)
+      super
+      @arguments = args
+    end
+  end
+end

data/lib/arrow/datum.rb

@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Datum
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Table
+          TableDatum.new(value)
+        when Array
+          ArrayDatum.new(value)
+        when ChunkedArray
+          ChunkedArrayDatum.new(value)
+        when Scalar
+          ScalarDatum.new(value)
+        when ::Array
+          ArrayDatum.new(ArrayBuilder.build(value))
+        when Integer
+          case value
+          when (0..((2 ** 8) - 1))
+            try_convert(UInt8Scalar.new(value))
+          when ((-(2 ** 7))..((2 ** 7) - 1))
+            try_convert(Int8Scalar.new(value))
+          when (0..((2 ** 16) - 1))
+            try_convert(UInt16Scalar.new(value))
+          when ((-(2 ** 15))..((2 ** 15) - 1))
+            try_convert(Int16Scalar.new(value))
+          when (0..((2 ** 32) - 1))
+            try_convert(UInt32Scalar.new(value))
+          when ((-(2 ** 31))..((2 ** 31) - 1))
+            try_convert(Int32Scalar.new(value))
+          when (0..((2 ** 64) - 1))
+            try_convert(UInt64Scalar.new(value))
+          when ((-(2 ** 63))..((2 ** 63) - 1))
+            try_convert(Int64Scalar.new(value))
+          else
+            nil
+          end
+        when Float
+          try_convert(DoubleScalar.new(value))
+        when true, false
+          try_convert(BooleanScalar.new(value))
+        when String
+          if value.ascii_only? or value.encoding == Encoding::UTF_8
+            if value.bytesize <= ((2 ** 31) - 1)
+              try_convert(StringScalar.new(value))
+            else
+              try_convert(LargeStringScalar.new(value))
+            end
+          else
+            if value.bytesize <= ((2 ** 31) - 1)
+              try_convert(BinaryScalar.new(value))
+            else
+              try_convert(LargeBinaryScalar.new(value))
+            end
+          end
+        when Date
+          date32_value = (value - Date32ArrayBuilder::UNIX_EPOCH).to_i
+          try_convert(Date32Scalar.new(date32_value))
+        when Time
+          case value.unit
+          when TimeUnit::SECOND, TimeUnit::MILLI
+            data_type = Time32DataType.new(value.unit)
+            scalar_class = Time32Scalar
+          else
+            data_type = Time64DataType.new(value.unit)
+            scalar_class = Time64Scalar
+          end
+          try_convert(scalar_class.new(data_type, value.value))
+        when ::Time
+          data_type = TimestampDataType.new(:nano)
+          timestamp_value = value.to_i * 1_000_000_000 + value.nsec
+          try_convert(TimestampScalar.new(data_type, timestamp_value))
+        when Decimal128
+          data_type = TimestampDataType.new(:nano)
+          timestamp_value = value.to_i * 1_000_000_000 + value.nsec
+          try_convert(Decimal128Scalar.new(data_type, timestamp_value))
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/equal-options.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class EqualOptions
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          options = new
+          value.each do |k, v|
+            setter = :"#{k}="
+            return unless options.respond_to?(setter)
+            options.__send__(setter, v)
+          end
+          options
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/expression.rb

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Expression
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Symbol
+          FieldExpression.new(value.to_s)
+        when ::Array
+          function_name, *arguments = value
+          case function_name
+          when String, Symbol
+            function_name = function_name.to_s
+          else
+            return nil
+          end
+          if arguments.last.is_a?(FunctionOptions)
+            options = arguments.pop
+          else
+            options = nil
+          end
+          CallExpression.new(function_name, arguments, options)
+        else
+          datum = Datum.try_convert(value)
+          return nil if datum.nil?
+          LiteralExpression.new(datum)
+        end
+      end
+    end
+  end
+end

data/lib/arrow/file-system.rb

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class FileSystem
+    alias_method :open_output_stream_raw, :open_output_stream
+    def open_output_stream(path)
+      stream = open_output_stream_raw(path)
+      if block_given?
+        begin
+          yield(stream)
+        ensure
+          stream.close
+        end
+      else
+        stream
+      end
+    end
+  end
+end