red-arrow 6.0.0 → 8.0.0

data/lib/arrow/table-loader.rb

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "uri"
+require "open-uri"
 
 module Arrow
   class TableLoader
@@ -34,30 +34,47 @@ module Arrow
 
     def load
       if @input.is_a?(URI)
-        custom_load_method = "load_from_uri"
+        custom_load_method_candidates = []
+        if @input.scheme
+          custom_load_method_candidates << "load_from_uri_#{@input.scheme}"
+        end
+        custom_load_method_candidates << "load_from_uri"
       elsif @input.is_a?(String) and ::File.directory?(@input)
-        custom_load_method = "load_from_directory"
+        custom_load_method_candidates = ["load_from_directory"]
       else
-        custom_load_method = "load_from_file"
+        custom_load_method_candidates = ["load_from_file"]
       end
-      unless respond_to?(custom_load_method, true)
-        available_schemes = []
-        (methods(true) | private_methods(true)).each do |name|
-          match_data = /\Aload_from_/.match(name.to_s)
-          if match_data
-            available_schemes << match_data.post_match
-          end
+      custom_load_method_candidates.each do |custom_load_method|
+        next unless respond_to?(custom_load_method, true)
+        return __send__(custom_load_method)
+      end
+      available_schemes = []
+      (methods(true) | private_methods(true)).each do |name|
+        match_data = /\Aload_from_/.match(name.to_s)
+        if match_data
+          available_schemes << match_data.post_match
         end
-        message = "Arrow::Table load source must be one of ["
-        message << available_schemes.join(", ")
-        message << "]: #{@input.inspect}"
-        raise ArgumentError, message
       end
-      __send__(custom_load_method)
+      message = "Arrow::Table load source must be one of ["
+      message << available_schemes.join(", ")
+      message << "]: #{@input.inspect}"
+      raise ArgumentError, message
     end
 
     private
+    def load_from_uri_http
+      load_by_reader
+    end
+
+    def load_from_uri_https
+      load_by_reader
+    end
+
     def load_from_file
+      load_by_reader
+    end
+
+    def load_by_reader
       format = @options[:format]
       custom_load_method = "load_as_#{format}"
       unless respond_to?(custom_load_method, true)
@@ -111,10 +128,29 @@ module Arrow
     end
 
     def open_input_stream
-      if @input.is_a?(Buffer)
-        BufferInputStream.new(@input)
+      case @input
+      when Buffer
+        yield(BufferInputStream.new(@input))
+      when URI
+        @input.open do |ruby_input|
+          case @options[:format]
+          when :stream, :arrow_streaming
+            Gio::RubyInputStream.open(ruby_input) do |gio_input|
+              GIOInputStream.open(gio_input) do |input|
+                yield(input)
+              end
+            end
+          else
+            # TODO: We need to consider Ruby's GVL carefully to use
+            # Ruby object directly for input with other formats. We
+            # read data and use it as Buffer for now.
+            data = GLib::Bytes.new(ruby_input.read.freeze)
+            buffer = Buffer.new(data)
+            yield(BufferInputStream.new(buffer))
+          end
+        end
       else
-        MemoryMappedInputStream.new(@input)
+        yield(MemoryMappedInputStream.new(@input))
       end
     end
 
@@ -130,32 +166,19 @@ module Arrow
     end
 
     def load_as_arrow
-      input = nil
-      reader = nil
-      error = nil
-      reader_class_candidates = [
-        RecordBatchFileReader,
-        RecordBatchStreamReader,
-      ]
-      reader_class_candidates.each do |reader_class_candidate|
-        input = open_input_stream
-        begin
-          reader = reader_class_candidate.new(input)
-        rescue Arrow::Error
-          error = $!
-        else
-          break
-        end
+      begin
+        load_as_arrow_file
+      rescue
+        load_as_arrows
       end
-      raise error if reader.nil?
-      load_raw(input, reader)
     end
 
     # @since 1.0.0
     def load_as_arrow_file
-      input = open_input_stream
-      reader = RecordBatchFileReader.new(input)
-      load_raw(input, reader)
+      open_input_stream do |input|
+        reader = RecordBatchFileReader.new(input)
+        load_raw(input, reader)
+      end
     end
 
     # @deprecated Use `format: :arrow_file` instead.
@@ -163,34 +186,46 @@ module Arrow
       load_as_arrow_file
     end
 
+    # @since 7.0.0
+    def load_as_arrows
+      open_input_stream do |input|
+        reader = RecordBatchStreamReader.new(input)
+        load_raw(input, reader)
+      end
+    end
+
     # @since 1.0.0
     def load_as_arrow_streaming
-      input = open_input_stream
-      reader = RecordBatchStreamReader.new(input)
-      load_raw(input, reader)
+      load_as_arrows
     end
 
     # @deprecated Use `format: :arrow_streaming` instead.
     def load_as_stream
-      load_as_arrow_streaming
+      load_as_arrows
     end
 
     if Arrow.const_defined?(:ORCFileReader)
       def load_as_orc
-        input = open_input_stream
-        reader = ORCFileReader.new(input)
-        field_indexes = @options[:field_indexes]
-        reader.set_field_indexes(field_indexes) if field_indexes
-        table = reader.read_stripes
-        table.instance_variable_set(:@input, input)
-        table
+        open_input_stream do |input|
+          reader = ORCFileReader.new(input)
+          field_indexes = @options[:field_indexes]
+          reader.set_field_indexes(field_indexes) if field_indexes
+          table = reader.read_stripes
+          table.instance_variable_set(:@input, input)
+          table
+        end
       end
     end
 
     def csv_load(options)
       options.delete(:format)
-      if @input.is_a?(Buffer)
+      case @input
+      when Buffer
         CSVLoader.load(@input.data.to_s, **options)
+      when URI
+        @input.open do |input|
+          CSVLoader.load(input.read, **options)
+        end
       else
         CSVLoader.load(Pathname.new(@input), **options)
       end
@@ -207,19 +242,21 @@ module Arrow
     end
 
     def load_as_feather
-      input = open_input_stream
-      reader = FeatherFileReader.new(input)
-      table = reader.read
-      table.instance_variable_set(:@input, input)
-      table
+      open_input_stream do |input|
+        reader = FeatherFileReader.new(input)
+        table = reader.read
+        table.instance_variable_set(:@input, input)
+        table
+      end
     end
 
     def load_as_json
-      input = open_input_stream
-      reader = JSONReader.new(input)
-      table = reader.read
-      table.instance_variable_set(:@input, input)
-      table
+      open_input_stream do |input|
+        reader = JSONReader.new(input)
+        table = reader.read
+        table.instance_variable_set(:@input, input)
+        table
+      end
     end
   end
 end

data/lib/arrow/table-saver.rb

@@ -151,14 +151,19 @@ module Arrow
       save_as_arrow_file
     end
 
+    # @since 7.0.0
+    def save_as_arrows
+      save_raw(RecordBatchStreamWriter)
+    end
+
     # @since 1.0.0
     def save_as_arrow_streaming
-      save_raw(RecordBatchStreamWriter)
+      save_as_arrows
     end
 
     # @deprecated Use `format: :arrow_streaming` instead.
     def save_as_stream
-      save_as_arrow_streaming
+      save_as_arrows
     end
 
     def csv_save(**options)

data/lib/arrow/table.rb

@@ -448,6 +448,84 @@ module Arrow
       self.class.new(schema, packed_arrays)
     end
 
+    # @overload join(right, key, type: :inner, left_outputs: nil, right_outputs: nil)
+    #   @!macro join_common_before
+    #     @param right [Arrow::Table] The right table.
+    #
+    #     Join columns with `right` on join key columns.
+    #
+    #   @!macro join_common_after
+    #     @param type [Arrow::JoinType] How to join.
+    #     @param left_outputs [::Array<String, Symbol>] Output columns in
+    #       `self`.
+    #
+    #       If both of `left_outputs` and `right_outputs` aren't
+    #       specified, all columns in `self` and `right` are
+    #       outputted.
+    #     @param right_outputs [::Array<String, Symbol>] Output columns in
+    #       `right`.
+    #
+    #       If both of `left_outputs` and `right_outputs` aren't
+    #       specified, all columns in `self` and `right` are
+    #       outputted.
+    #     @return [Arrow::Table]
+    #       The joined `Arrow::Table`.
+    #
+    #   @macro join_common_before
+    #   @param key [String, Symbol] A join key.
+    #   @macro join_common_after
+    #
+    # @overload join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+    #
+    #   @macro join_common_before
+    #   @param keys [::Array<String, Symbol>] Join keys.
+    #   @macro join_common_after
+    #
+    # @overload join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+    #
+    #   @macro join_common_before
+    #   @param keys [Hash] Specify join keys in `self` and `right` separately.
+    #   @option keys [String, Symbol, ::Array<String, Symbol>] :left
+    #     Join keys in `self`.
+    #   @option keys [String, Symbol, ::Array<String, Symbol>] :right
+    #     Join keys in `right`.
+    #   @macro join_common_after
+    #
+    # @since 7.0.0
+    def join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+      plan = ExecutePlan.new
+      left_node = plan.build_source_node(self)
+      right_node = plan.build_source_node(right)
+      if keys.is_a?(Hash)
+        left_keys = keys[:left]
+        right_keys = keys[:right]
+      else
+        left_keys = keys
+        right_keys = keys
+      end
+      left_keys = Array(left_keys)
+      right_keys = Array(right_keys)
+      hash_join_node_options = HashJoinNodeOptions.new(type,
+                                                       left_keys,
+                                                       right_keys)
+      unless left_outputs.nil?
+        hash_join_node_options.left_outputs = left_outputs
+      end
+      unless right_outputs.nil?
+        hash_join_node_options.right_outputs = right_outputs
+      end
+      hash_join_node = plan.build_hash_join_node(left_node,
+                                                 right_node,
+                                                 hash_join_node_options)
+      sink_node_options = SinkNodeOptions.new
+      plan.build_sink_node(hash_join_node, sink_node_options)
+      plan.validate
+      plan.start
+      plan.wait
+      reader = sink_node_options.get_reader(hash_join_node.output_schema)
+      reader.read_all
+    end
+
     alias_method :to_s_raw, :to_s
     def to_s(options={})
       format = options[:format]

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "6.0.0"
+  VERSION = "8.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -48,19 +48,10 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency("bigdecimal", ">= 2.0.3")
   spec.add_runtime_dependency("extpp", ">= 0.0.7")
-  spec.add_runtime_dependency("gio2", ">= 3.4.9")
+  spec.add_runtime_dependency("gio2", ">= 3.5.0")
   spec.add_runtime_dependency("native-package-installer")
   spec.add_runtime_dependency("pkg-config")
 
-  spec.add_development_dependency("benchmark-driver")
-  spec.add_development_dependency("bundler")
-  spec.add_development_dependency("faker")
-  spec.add_development_dependency("fiddle", ">= 1.0.9")
-  spec.add_development_dependency("rake")
-  spec.add_development_dependency("redcarpet")
-  spec.add_development_dependency("test-unit")
-  spec.add_development_dependency("yard")
-
   required_msys2_package_version = version_components[0, 3].join(".")
   spec.metadata["msys2_mingw_dependencies"] =
     "arrow>=#{required_msys2_package_version}"

data/test/helper.rb

@@ -20,6 +20,8 @@ require "arrow"
 require "fiddle"
 require "pathname"
 require "tempfile"
+require "timeout"
+require "webrick"
 require "zlib"
 
 require "test-unit"

data/test/raw-records/test-basic-arrays.rb

@@ -346,6 +346,36 @@ module RawRecordsBasicArraysTests
                    records)
     assert_equal(records, target.raw_records)
   end
+
+  def test_month_interval
+    records = [
+      [1],
+      [nil],
+      [12],
+    ]
+    target = build({column: :month_interval}, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{day: 1, millisecond: 100}],
+      [nil],
+      [{day: 2, millisecond: 300}],
+    ]
+    target = build({column: :day_time_interval}, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{month: 1, day: 1, nanosecond: 100}],
+      [nil],
+      [{month: 2, day: 3, nanosecond: 400}],
+    ]
+    target = build({column: :month_day_nano_interval}, records)
+    assert_equal(records, target.raw_records)
+  end
 end
 
 class RawRecordsRecordBatchBasicArraysTest < Test::Unit::TestCase

data/test/raw-records/test-dense-union-array.rb

@@ -359,6 +359,33 @@ module RawRecordsDenseUnionArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [{"0" => 1}],
+      [{"1" => nil}],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{"0" => {day: 1, millisecond: 100}}],
+      [{"1" => nil}],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{"0" => {month: 1, day: 1, nanosecond: 100}}],
+      [{"1" => nil}],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"0" => [true, nil, false]}],

data/test/raw-records/test-list-array.rb

@@ -399,6 +399,45 @@ module RawRecordsListArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [[1, nil, 12]],
+      [nil],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [
+        [
+          {day: 1, millisecond: 100},
+          nil,
+          {day: 2, millisecond: 300},
+        ]
+      ],
+      [nil],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [
+        [
+          {month: 1, day: 1, nanosecond: 100},
+          nil,
+          {month: 2, day: 3, nanosecond: 400},
+        ]
+      ],
+      [nil],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [

data/test/raw-records/test-map-array.rb

@@ -310,6 +310,43 @@ module RawRecordsMapArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [{"key1" => 1, "key2" => nil}],
+      [nil],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [
+        {
+          "key1" => {day: 1, millisecond: 100},
+          "key2" => nil,
+        },
+      ],
+      [nil],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [
+        {
+          "key1" => {month: 1, day: 1, nanosecond: 100},
+          "key2" => nil,
+        },
+      ],
+      [nil],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"key1" => [true, nil, false], "key2" => nil}],

data/test/raw-records/test-sparse-union-array.rb

@@ -349,6 +349,33 @@ module RawRecordsSparseUnionArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [{"0" => 1}],
+      [{"1" => nil}],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{"0" => {day: 1, millisecond: 100}}],
+      [{"1" => nil}],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{"0" => {month: 1, day: 1, nanosecond: 100}}],
+      [{"1" => nil}],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"0" => [true, nil, false]}],

data/test/raw-records/test-struct-array.rb

@@ -344,6 +344,36 @@ module RawRecordsStructArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_month_interval
+    records = [
+      [{"field" => 1}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:month_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_day_time_interval
+    records = [
+      [{"field" => {day: 1, millisecond: 100}}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:day_time_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
+  def test_month_day_nano_interval
+    records = [
+      [{"field" => {month: 1, day: 1, nanosecond: 100}}],
+      [nil],
+      [{"field" => nil}],
+    ]
+    target = build(:month_day_nano_interval, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_list
     records = [
       [{"field" => [true, nil, false]}],