red-arrow 6.0.1 → 9.0.0

data/lib/arrow/table-loader.rb

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "uri"
+require "open-uri"
 
 module Arrow
   class TableLoader
@@ -34,30 +34,47 @@ module Arrow
 
     def load
       if @input.is_a?(URI)
-        custom_load_method = "load_from_uri"
+        custom_load_method_candidates = []
+        if @input.scheme
+          custom_load_method_candidates << "load_from_uri_#{@input.scheme}"
+        end
+        custom_load_method_candidates << "load_from_uri"
       elsif @input.is_a?(String) and ::File.directory?(@input)
-        custom_load_method = "load_from_directory"
+        custom_load_method_candidates = ["load_from_directory"]
       else
-        custom_load_method = "load_from_file"
+        custom_load_method_candidates = ["load_from_file"]
       end
-      unless respond_to?(custom_load_method, true)
-        available_schemes = []
-        (methods(true) | private_methods(true)).each do |name|
-          match_data = /\Aload_from_/.match(name.to_s)
-          if match_data
-            available_schemes << match_data.post_match
-          end
+      custom_load_method_candidates.each do |custom_load_method|
+        next unless respond_to?(custom_load_method, true)
+        return __send__(custom_load_method)
+      end
+      available_schemes = []
+      (methods(true) | private_methods(true)).each do |name|
+        match_data = /\Aload_from_/.match(name.to_s)
+        if match_data
+          available_schemes << match_data.post_match
         end
-        message = "Arrow::Table load source must be one of ["
-        message << available_schemes.join(", ")
-        message << "]: #{@input.inspect}"
-        raise ArgumentError, message
       end
-      __send__(custom_load_method)
+      message = "Arrow::Table load source must be one of ["
+      message << available_schemes.join(", ")
+      message << "]: #{@input.inspect}"
+      raise ArgumentError, message
     end
 
     private
+    def load_from_uri_http
+      load_by_reader
+    end
+
+    def load_from_uri_https
+      load_by_reader
+    end
+
     def load_from_file
+      load_by_reader
+    end
+
+    def load_by_reader
       format = @options[:format]
       custom_load_method = "load_as_#{format}"
       unless respond_to?(custom_load_method, true)
@@ -111,10 +128,29 @@ module Arrow
     end
 
     def open_input_stream
-      if @input.is_a?(Buffer)
-        BufferInputStream.new(@input)
+      case @input
+      when Buffer
+        yield(BufferInputStream.new(@input))
+      when URI
+        @input.open do |ruby_input|
+          case @options[:format]
+          when :stream, :arrow_streaming
+            Gio::RubyInputStream.open(ruby_input) do |gio_input|
+              GIOInputStream.open(gio_input) do |input|
+                yield(input)
+              end
+            end
+          else
+            # TODO: We need to consider Ruby's GVL carefully to use
+            # Ruby object directly for input with other formats. We
+            # read data and use it as Buffer for now.
+            data = GLib::Bytes.new(ruby_input.read.freeze)
+            buffer = Buffer.new(data)
+            yield(BufferInputStream.new(buffer))
+          end
+        end
       else
-        MemoryMappedInputStream.new(@input)
+        yield(MemoryMappedInputStream.new(@input))
       end
     end
 
@@ -130,32 +166,19 @@ module Arrow
     end
 
     def load_as_arrow
-      input = nil
-      reader = nil
-      error = nil
-      reader_class_candidates = [
-        RecordBatchFileReader,
-        RecordBatchStreamReader,
-      ]
-      reader_class_candidates.each do |reader_class_candidate|
-        input = open_input_stream
-        begin
-          reader = reader_class_candidate.new(input)
-        rescue Arrow::Error
-          error = $!
-        else
-          break
-        end
+      begin
+        load_as_arrow_file
+      rescue
+        load_as_arrows
       end
-      raise error if reader.nil?
-      load_raw(input, reader)
     end
 
     # @since 1.0.0
     def load_as_arrow_file
-      input = open_input_stream
-      reader = RecordBatchFileReader.new(input)
-      load_raw(input, reader)
+      open_input_stream do |input|
+        reader = RecordBatchFileReader.new(input)
+        load_raw(input, reader)
+      end
     end
 
     # @deprecated Use `format: :arrow_file` instead.
@@ -163,34 +186,46 @@ module Arrow
       load_as_arrow_file
     end
 
+    # @since 7.0.0
+    def load_as_arrows
+      open_input_stream do |input|
+        reader = RecordBatchStreamReader.new(input)
+        load_raw(input, reader)
+      end
+    end
+
     # @since 1.0.0
     def load_as_arrow_streaming
-      input = open_input_stream
-      reader = RecordBatchStreamReader.new(input)
-      load_raw(input, reader)
+      load_as_arrows
     end
 
     # @deprecated Use `format: :arrow_streaming` instead.
     def load_as_stream
-      load_as_arrow_streaming
+      load_as_arrows
     end
 
     if Arrow.const_defined?(:ORCFileReader)
       def load_as_orc
-        input = open_input_stream
-        reader = ORCFileReader.new(input)
-        field_indexes = @options[:field_indexes]
-        reader.set_field_indexes(field_indexes) if field_indexes
-        table = reader.read_stripes
-        table.instance_variable_set(:@input, input)
-        table
+        open_input_stream do |input|
+          reader = ORCFileReader.new(input)
+          field_indexes = @options[:field_indexes]
+          reader.set_field_indexes(field_indexes) if field_indexes
+          table = reader.read_stripes
+          table.instance_variable_set(:@input, input)
+          table
+        end
       end
     end
 
     def csv_load(options)
       options.delete(:format)
-      if @input.is_a?(Buffer)
+      case @input
+      when Buffer
         CSVLoader.load(@input.data.to_s, **options)
+      when URI
+        @input.open do |input|
+          CSVLoader.load(input.read, **options)
+        end
       else
         CSVLoader.load(Pathname.new(@input), **options)
       end
@@ -207,19 +242,21 @@ module Arrow
     end
 
     def load_as_feather
-      input = open_input_stream
-      reader = FeatherFileReader.new(input)
-      table = reader.read
-      table.instance_variable_set(:@input, input)
-      table
+      open_input_stream do |input|
+        reader = FeatherFileReader.new(input)
+        table = reader.read
+        table.instance_variable_set(:@input, input)
+        table
+      end
     end
 
     def load_as_json
-      input = open_input_stream
-      reader = JSONReader.new(input)
-      table = reader.read
-      table.instance_variable_set(:@input, input)
-      table
+      open_input_stream do |input|
+        reader = JSONReader.new(input)
+        table = reader.read
+        table.instance_variable_set(:@input, input)
+        table
+      end
     end
   end
 end

data/lib/arrow/table-saver.rb

@@ -151,14 +151,19 @@ module Arrow
       save_as_arrow_file
     end
 
+    # @since 7.0.0
+    def save_as_arrows
+      save_raw(RecordBatchStreamWriter)
+    end
+
     # @since 1.0.0
     def save_as_arrow_streaming
-      save_raw(RecordBatchStreamWriter)
+      save_as_arrows
     end
 
     # @deprecated Use `format: :arrow_streaming` instead.
     def save_as_stream
-      save_as_arrow_streaming
+      save_as_arrows
     end
 
     def csv_save(**options)

data/lib/arrow/table.rb

@@ -448,6 +448,84 @@ module Arrow
       self.class.new(schema, packed_arrays)
     end
 
+    # @overload join(right, key, type: :inner, left_outputs: nil, right_outputs: nil)
+    #   @!macro join_common_before
+    #     @param right [Arrow::Table] The right table.
+    #
+    #     Join columns with `right` on join key columns.
+    #
+    #   @!macro join_common_after
+    #     @param type [Arrow::JoinType] How to join.
+    #     @param left_outputs [::Array<String, Symbol>] Output columns in
+    #       `self`.
+    #
+    #       If both of `left_outputs` and `right_outputs` aren't
+    #       specified, all columns in `self` and `right` are
+    #       outputted.
+    #     @param right_outputs [::Array<String, Symbol>] Output columns in
+    #       `right`.
+    #
+    #       If both of `left_outputs` and `right_outputs` aren't
+    #       specified, all columns in `self` and `right` are
+    #       outputted.
+    #     @return [Arrow::Table]
+    #       The joined `Arrow::Table`.
+    #
+    #   @macro join_common_before
+    #   @param key [String, Symbol] A join key.
+    #   @macro join_common_after
+    #
+    # @overload join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+    #
+    #   @macro join_common_before
+    #   @param keys [::Array<String, Symbol>] Join keys.
+    #   @macro join_common_after
+    #
+    # @overload join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+    #
+    #   @macro join_common_before
+    #   @param keys [Hash] Specify join keys in `self` and `right` separately.
+    #   @option keys [String, Symbol, ::Array<String, Symbol>] :left
+    #     Join keys in `self`.
+    #   @option keys [String, Symbol, ::Array<String, Symbol>] :right
+    #     Join keys in `right`.
+    #   @macro join_common_after
+    #
+    # @since 7.0.0
+    def join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+      plan = ExecutePlan.new
+      left_node = plan.build_source_node(self)
+      right_node = plan.build_source_node(right)
+      if keys.is_a?(Hash)
+        left_keys = keys[:left]
+        right_keys = keys[:right]
+      else
+        left_keys = keys
+        right_keys = keys
+      end
+      left_keys = Array(left_keys)
+      right_keys = Array(right_keys)
+      hash_join_node_options = HashJoinNodeOptions.new(type,
+                                                       left_keys,
+                                                       right_keys)
+      unless left_outputs.nil?
+        hash_join_node_options.left_outputs = left_outputs
+      end
+      unless right_outputs.nil?
+        hash_join_node_options.right_outputs = right_outputs
+      end
+      hash_join_node = plan.build_hash_join_node(left_node,
+                                                 right_node,
+                                                 hash_join_node_options)
+      sink_node_options = SinkNodeOptions.new
+      plan.build_sink_node(hash_join_node, sink_node_options)
+      plan.validate
+      plan.start
+      plan.wait
+      reader = sink_node_options.get_reader(hash_join_node.output_schema)
+      reader.read_all
+    end
+
     alias_method :to_s_raw, :to_s
     def to_s(options={})
       format = options[:format]

data/lib/arrow/time-unit.rb

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class TimeUnit
+    class << self
+      # @api private
+      def try_convert(value)
+        if value.is_a?(Hash) and value.size == 1 and value[:unit]
+          super(value[:unit])
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/arrow/time32-array-builder.rb

@@ -18,24 +18,12 @@
 module Arrow
   class Time32ArrayBuilder
     class << self
-      def build(unit_or_data_type, values)
-        builder = new(unit_or_data_type)
+      def build(data_type, values)
+        builder = new(data_type)
         builder.build(values)
       end
     end
 
-    alias_method :initialize_raw, :initialize
-    def initialize(unit_or_data_type)
-      case unit_or_data_type
-      when DataType
-        data_type = unit_or_data_type
-      else
-        unit = unit_or_data_type
-        data_type = Time32DataType.new(unit)
-      end
-      initialize_raw(data_type)
-    end
-
     def unit
       @unit ||= value_data_type.unit
     end

data/lib/arrow/time32-data-type.rb

@@ -17,45 +17,16 @@
 
 module Arrow
   class Time32DataType
-    alias_method :initialize_raw, :initialize
-    private :initialize_raw
-
-    # Creates a new {Arrow::Time32DataType}.
-    #
-    # @overload initialize(unit)
-    #
-    #   @param unit [Arrow::TimeUnit, Symbol] The unit of the
-    #     time32 data type.
-    #
-    #     The unit must be second or millisecond.
-    #
-    #   @example Create a time32 data type with Arrow::TimeUnit
-    #     Arrow::Time32DataType.new(Arrow::TimeUnit::MILLI)
-    #
-    #   @example Create a time32 data type with Symbol
-    #     Arrow::Time32DataType.new(:milli)
-    #
-    # @overload initialize(description)
-    #
-    #   @param description [Hash] The description of the time32 data
-    #     type. It must have `:unit` value.
-    #
-    #   @option description [Arrow::TimeUnit, Symbol] :unit The unit of
-    #     the time32 data type.
-    #
-    #     The unit must be second or millisecond.
-    #
-    #   @example Create a time32 data type with Arrow::TimeUnit
-    #     Arrow::Time32DataType.new(unit: Arrow::TimeUnit::MILLI)
-    #
-    #   @example Create a time32 data type with Symbol
-    #     Arrow::Time32DataType.new(unit: :milli)
-    def initialize(unit)
-      if unit.is_a?(Hash)
-        description = unit
-        unit = description[:unit]
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Symbol, Arrow::TimeUnit
+          new(value)
+        else
+          super
+        end
       end
-      initialize_raw(unit)
     end
   end
 end

data/lib/arrow/time64-array-builder.rb

@@ -18,24 +18,12 @@
 module Arrow
   class Time64ArrayBuilder
     class << self
-      def build(unit_or_data_type, values)
-        builder = new(unit_or_data_type)
+      def build(data_type, values)
+        builder = new(data_type)
         builder.build(values)
       end
     end
 
-    alias_method :initialize_raw, :initialize
-    def initialize(unit_or_data_type)
-      case unit_or_data_type
-      when DataType
-        data_type = unit_or_data_type
-      else
-        unit = unit_or_data_type
-        data_type = Time64DataType.new(unit)
-      end
-      initialize_raw(data_type)
-    end
-
     def unit
       @unit ||= value_data_type.unit
     end

data/lib/arrow/time64-data-type.rb

@@ -17,45 +17,16 @@
 
 module Arrow
   class Time64DataType
-    alias_method :initialize_raw, :initialize
-    private :initialize_raw
-
-    # Creates a new {Arrow::Time64DataType}.
-    #
-    # @overload initialize(unit)
-    #
-    #   @param unit [Arrow::TimeUnit, Symbol] The unit of the
-    #     time64 data type.
-    #
-    #     The unit must be microsecond or nanosecond.
-    #
-    #   @example Create a time64 data type with Arrow::TimeUnit
-    #     Arrow::Time64DataType.new(Arrow::TimeUnit::NANO)
-    #
-    #   @example Create a time64 data type with Symbol
-    #     Arrow::Time64DataType.new(:nano)
-    #
-    # @overload initialize(description)
-    #
-    #   @param description [Hash] The description of the time64 data
-    #     type. It must have `:unit` value.
-    #
-    #   @option description [Arrow::TimeUnit, Symbol] :unit The unit of
-    #     the time64 data type.
-    #
-    #     The unit must be microsecond or nanosecond.
-    #
-    #   @example Create a time64 data type with Arrow::TimeUnit
-    #     Arrow::Time64DataType.new(unit: Arrow::TimeUnit::NANO)
-    #
-    #   @example Create a time64 data type with Symbol
-    #     Arrow::Time64DataType.new(unit: :nano)
-    def initialize(unit)
-      if unit.is_a?(Hash)
-        description = unit
-        unit = description[:unit]
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Symbol, Arrow::TimeUnit
+          new(value)
+        else
+          super
+        end
       end
-      initialize_raw(unit)
     end
   end
 end

data/lib/arrow/timestamp-array-builder.rb

@@ -18,24 +18,12 @@
 module Arrow
   class TimestampArrayBuilder
     class << self
-      def build(unit_or_data_type, values)
-        builder = new(unit_or_data_type)
+      def build(data_type, values)
+        builder = new(data_type)
         builder.build(values)
       end
     end
 
-    alias_method :initialize_raw, :initialize
-    def initialize(unit_or_data_type)
-      case unit_or_data_type
-      when DataType
-        data_type = unit_or_data_type
-      else
-        unit = unit_or_data_type
-        data_type = TimestampDataType.new(unit)
-      end
-      initialize_raw(data_type)
-    end
-
     private
     def unit_id
       @unit_id ||= value_data_type.unit.nick.to_sym

data/lib/arrow/timestamp-data-type.rb

@@ -17,41 +17,16 @@
 
 module Arrow
   class TimestampDataType
-    alias_method :initialize_raw, :initialize
-    private :initialize_raw
-
-    # Creates a new {Arrow::TimestampDataType}.
-    #
-    # @overload initialize(unit)
-    #
-    #   @param unit [Arrow::TimeUnit, Symbol] The unit of the
-    #     timestamp data type.
-    #
-    #   @example Create a timestamp data type with Arrow::TimeUnit
-    #     Arrow::TimestampDataType.new(Arrow::TimeUnit::MILLI)
-    #
-    #   @example Create a timestamp data type with Symbol
-    #     Arrow::TimestampDataType.new(:milli)
-    #
-    # @overload initialize(description)
-    #
-    #   @param description [Hash] The description of the timestamp data
-    #     type. It must have `:unit` value.
-    #
-    #   @option description [Arrow::TimeUnit, Symbol] :unit The unit of
-    #     the timestamp data type.
-    #
-    #   @example Create a timestamp data type with Arrow::TimeUnit
-    #     Arrow::TimestampDataType.new(unit: Arrow::TimeUnit::MILLI)
-    #
-    #   @example Create a timestamp data type with Symbol
-    #     Arrow::TimestampDataType.new(unit: :milli)
-    def initialize(unit)
-      if unit.is_a?(Hash)
-        description = unit
-        unit = description[:unit]
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Symbol, Arrow::TimeUnit
+          new(value)
+        else
+          super
+        end
       end
-      initialize_raw(unit)
     end
   end
 end

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "6.0.1"
+  VERSION = "9.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -46,21 +46,12 @@ Gem::Specification.new do |spec|
   spec.test_files += Dir.glob("test/**/*")
   spec.extensions = ["ext/arrow/extconf.rb"]
 
-  spec.add_runtime_dependency("bigdecimal", ">= 2.0.3")
+  spec.add_runtime_dependency("bigdecimal", ">= 3.1.0")
   spec.add_runtime_dependency("extpp", ">= 0.0.7")
-  spec.add_runtime_dependency("gio2", ">= 3.4.9")
+  spec.add_runtime_dependency("gio2", ">= 3.5.0")
   spec.add_runtime_dependency("native-package-installer")
   spec.add_runtime_dependency("pkg-config")
 
-  spec.add_development_dependency("benchmark-driver")
-  spec.add_development_dependency("bundler")
-  spec.add_development_dependency("faker")
-  spec.add_development_dependency("fiddle", ">= 1.0.9")
-  spec.add_development_dependency("rake")
-  spec.add_development_dependency("redcarpet")
-  spec.add_development_dependency("test-unit")
-  spec.add_development_dependency("yard")
-
   required_msys2_package_version = version_components[0, 3].join(".")
   spec.metadata["msys2_mingw_dependencies"] =
     "arrow>=#{required_msys2_package_version}"

data/test/helper.rb

@@ -20,6 +20,8 @@ require "arrow"
 require "fiddle"
 require "pathname"
 require "tempfile"
+require "timeout"
+require "webrick"
 require "zlib"
 
 require "test-unit"