red-arrow 4.0.0 → 6.0.1

data/lib/arrow/table-loader.rb

@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require "uri"
+
 module Arrow
   class TableLoader
     class << self
@@ -31,6 +33,31 @@ module Arrow
     end
 
     def load
+      if @input.is_a?(URI)
+        custom_load_method = "load_from_uri"
+      elsif @input.is_a?(String) and ::File.directory?(@input)
+        custom_load_method = "load_from_directory"
+      else
+        custom_load_method = "load_from_file"
+      end
+      unless respond_to?(custom_load_method, true)
+        available_schemes = []
+        (methods(true) | private_methods(true)).each do |name|
+          match_data = /\Aload_from_/.match(name.to_s)
+          if match_data
+            available_schemes << match_data.post_match
+          end
+        end
+        message = "Arrow::Table load source must be one of ["
+        message << available_schemes.join(", ")
+        message << "]: #{@input.inspect}"
+        raise ArgumentError, message
+      end
+      __send__(custom_load_method)
+    end
+
+    private
+    def load_from_file
       format = @options[:format]
       custom_load_method = "load_as_#{format}"
       unless respond_to?(custom_load_method, true)
@@ -56,21 +83,24 @@ module Arrow
       end
     end
 
-    private
     def fill_options
       if @options[:format] and @options.key?(:compression)
         return
       end
 
-      if @input.is_a?(Buffer)
+      case @input
+      when Buffer
         info = {}
+      when URI
+        extension = PathExtension.new(@input.path)
+        info = extension.extract
       else
         extension = PathExtension.new(@input)
         info = extension.extract
       end
       format = info[:format]
       @options = @options.dup
-      if format and respond_to?("load_as_#{format}", true)
+      if format
         @options[:format] ||= format.to_sym
       else
         @options[:format] ||= :arrow
@@ -183,5 +213,13 @@ module Arrow
       table.instance_variable_set(:@input, input)
       table
     end
+
+    def load_as_json
+      input = open_input_stream
+      reader = JSONReader.new(input)
+      table = reader.read
+      table.instance_variable_set(:@input, input)
+      table
+    end
   end
 end

data/lib/arrow/table-saver.rb

@@ -32,6 +32,29 @@ module Arrow
     end
 
     def save
+      if @output.is_a?(URI)
+        custom_save_method = "save_to_uri"
+      else
+        custom_save_method = "save_to_file"
+      end
+      unless respond_to?(custom_save_method, true)
+        available_schemes = []
+        (methods(true) | private_methods(true)).each do |name|
+          match_data = /\Asave_to_/.match(name.to_s)
+          if match_data
+            available_schemes << match_data.post_match
+          end
+        end
+        message = "Arrow::Table save source must be one of ["
+        message << available_schemes.join(", ")
+        message << "]: #{@output.scheme.inspect}"
+        raise ArgumentError, message
+      end
+      __send__(custom_save_method)
+    end
+
+    private
+    def save_to_file
       format = @options[:format]
       custom_save_method = "save_as_#{format}"
       unless respond_to?(custom_save_method, true)
@@ -57,21 +80,24 @@ module Arrow
       end
     end
 
-    private
     def fill_options
       if @options[:format] and @options.key?(:compression)
         return
       end
 
-      if @output.is_a?(Buffer)
+      case @output
+      when Buffer
         info = {}
+      when URI
+        extension = PathExtension.new(@output.path)
+        info = extension.extract
       else
         extension = PathExtension.new(@output)
         info = extension.extract
       end
       format = info[:format]
       @options = @options.dup
-      if format and respond_to?("save_as_#{format}", true)
+      if format
         @options[:format] ||= format.to_sym
       else
         @options[:format] ||= :arrow

data/lib/arrow/table-table-formatter.rb

@@ -21,51 +21,27 @@ module Arrow
   # TODO: Almost codes should be implemented in Apache Arrow C++.
   class TableTableFormatter < TableFormatter
     private
-    def format_header(text, columns)
-      columns.each do |column|
+    def format_header(text, column_formatters)
+      column_formatters.each do |column_formatter|
         text << "\t"
-        text << format_column_name(column)
+        text << column_formatter.aligned_name
       end
       text << "\n"
     end
 
-    FLOAT_N_DIGITS = 10
-    def format_column_name(column)
-      case column.data_type
-      when TimestampDataType
-        "%*s" % [::Time.now.iso8601.size, column.name]
-      when FloatDataType, DoubleDataType
-        "%*s" % [FLOAT_N_DIGITS, column.name]
-      else
-        column.name
-      end
-    end
-
-    def format_rows(text, columns, rows, n_digits, start_offset)
+    def format_rows(text, column_formatters, rows, n_digits, start_offset)
       rows.each_with_index do |row, nth_row|
         text << ("%*d" % [n_digits, start_offset + nth_row])
         row.each_with_index do |column_value, nth_column|
           text << "\t"
-          column = columns[nth_column]
-          text << format_column_value(column, column_value)
+          column_formatter = column_formatters[nth_column]
+          aligned_name = column_formatter.aligned_name
+          text << column_formatter.format_value(column_value, aligned_name.size)
         end
         text << "\n"
       end
     end
 
-    def format_column_value(column, value)
-      case value
-      when ::Time
-        value.iso8601
-      when Float
-        "%*f" % [[column.name.size, FLOAT_N_DIGITS].max, value]
-      when Integer
-        "%*d" % [column.name.size, value]
-      else
-        "%-*s" % [column.name.size, value.to_s]
-      end
-    end
-
     def format_ellipsis(text)
       text << "...\n"
     end

data/lib/arrow/table.rb

@@ -195,8 +195,6 @@ module Arrow
     alias_method :size, :n_rows
     alias_method :length, :n_rows
 
-    alias_method :[], :find_column
-
     alias_method :slice_raw, :slice
 
     # @overload slice(offset, length)
@@ -236,6 +234,12 @@ module Arrow
     #   @return [Arrow::Table]
     #     The sub `Arrow::Table` that covers only rows of the range of indices.
     #
+    # @overload slice(conditions)
+    #
+    #   @param conditions [Hash] The conditions to select records.
+    #   @return [Arrow::Table]
+    #     The sub `Arrow::Table` that covers only rows matched by condition
+    #
     # @overload slice
     #
     #   @yield [slicer] Gives slicer that constructs condition to select records.
@@ -263,12 +267,37 @@ module Arrow
         expected_n_args = nil
         case args.size
         when 1
-          if args[0].is_a?(Integer)
+          case args[0]
+          when Integer
             index = args[0]
             index += n_rows if index < 0
             return nil if index < 0
             return nil if index >= n_rows
             return Record.new(self, index)
+          when Hash
+            condition_pairs = args[0]
+            slicer = Slicer.new(self)
+            conditions = []
+            condition_pairs.each do |key, value|
+              case value
+              when Range
+                # TODO: Optimize "begin <= key <= end" case by missing "between" kernel
+                # https://issues.apache.org/jira/browse/ARROW-9843
+                unless value.begin.nil?
+                  conditions << (slicer[key] >= value.begin)
+                end
+                unless value.end.nil?
+                  if value.exclude_end?
+                    conditions << (slicer[key] < value.end)
+                  else
+                    conditions << (slicer[key] <= value.end)
+                  end
+                end
+              else
+                conditions << (slicer[key] == value)
+              end
+            end
+            slicers << conditions.inject(:&)
           else
             slicers << args[0]
           end
@@ -397,41 +426,6 @@ module Arrow
       remove_column_raw(index)
     end
 
-    # TODO
-    #
-    # @return [Arrow::Table]
-    def select_columns(*selectors, &block)
-      if selectors.empty?
-        return to_enum(__method__) unless block_given?
-        selected_columns = columns.select(&block)
-      else
-        selected_columns = []
-        selectors.each do |selector|
-          case selector
-          when String, Symbol
-            column = find_column(selector)
-            if column.nil?
-              message = "unknown column: #{selector.inspect}: #{inspect}"
-              raise KeyError.new(message)
-            end
-            selected_columns << column
-          when Range
-            selected_columns.concat(columns[selector])
-          else
-            column = columns[selector]
-            if column.nil?
-              message = "out of index (0..#{n_columns - 1}): " +
-              "#{selector.inspect}: #{inspect}"
-              raise IndexError.new(message)
-            end
-            selected_columns << column
-          end
-        end
-        selected_columns = selected_columns.select(&block) if block_given?
-      end
-      self.class.new(selected_columns)
-    end
-
     # Experimental
     def group(*keys)
       Group.new(self, keys)
@@ -442,8 +436,8 @@ module Arrow
       RollingWindow.new(self, size)
     end
 
-    def save(path, options={})
-      saver = TableSaver.new(self, path, options)
+    def save(output, options={})
+      saver = TableSaver.new(self, output, options)
       saver.save
     end
 

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "4.0.0"
+  VERSION = "6.0.1"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -48,13 +48,14 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency("bigdecimal", ">= 2.0.3")
   spec.add_runtime_dependency("extpp", ">= 0.0.7")
-  spec.add_runtime_dependency("gio2", ">= 3.3.6")
+  spec.add_runtime_dependency("gio2", ">= 3.4.9")
   spec.add_runtime_dependency("native-package-installer")
   spec.add_runtime_dependency("pkg-config")
 
   spec.add_development_dependency("benchmark-driver")
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("faker")
+  spec.add_development_dependency("fiddle", ">= 1.0.9")
   spec.add_development_dependency("rake")
   spec.add_development_dependency("redcarpet")
   spec.add_development_dependency("test-unit")

data/test/helper.rb

@@ -17,6 +17,7 @@
 
 require "arrow"
 
+require "fiddle"
 require "pathname"
 require "tempfile"
 require "zlib"

data/test/raw-records/test-dense-union-array.rb

@@ -394,6 +394,20 @@ module RawRecordsDenseUnionArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_map
+    records = [
+      [{"0" => {"key1" => true, "key2" => nil}}],
+      [{"1" => nil}],
+    ]
+    target = build({
+                     type: :map,
+                     key: :string,
+                     item: :boolean,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_sparse_union
     omit("Need to add support for SparseUnionArrayBuilder")
     records = [

data/test/raw-records/test-list-array.rb

@@ -451,6 +451,25 @@ module RawRecordsListArrayTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_map
+    records = [
+      [
+        [
+          {"key1" => true, "key2" => nil},
+          nil,
+        ],
+      ],
+      [nil],
+    ]
+    target = build({
+                     type: :map,
+                     key: :string,
+                     item: :boolean,
+                   },
+                   records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_sparse
     omit("Need to add support for SparseUnionArrayBuilder")
     records = [