red-arrow 0.15.1 → 0.16.0

data/lib/arrow/list-data-type.rb

@@ -53,16 +53,66 @@ module Arrow
     #
     #   @example Create a list data type with field description
     #     Arrow::ListDataType.new(field: {name: "visible", type: :boolean})
-    def initialize(field)
-      if field.is_a?(Hash) and field.key?(:field)
-        description = field
-        field = description[:field]
-      end
-      if field.is_a?(Hash)
-        field_description = field
-        field = Field.new(field_description)
+    #
+    # @overload initialize(data_type)
+    #
+    #   @param data_type [Arrow::DataType, String, Symbol,
+    #     ::Array<String>, ::Array<Symbol>, Hash] The element data
+    #     type of the list data type. A field is created with the
+    #     default name `"item"` from the data type automatically.
+    #
+    #     See {Arrow::DataType.resolve} how to specify data type.
+    #
+    #   @example Create a list data type with {Arrow::DataType}
+    #     Arrow::ListDataType.new(Arrow::BooleanDataType.new)
+    #
+    #   @example Create a list data type with data type name as String
+    #     Arrow::ListDataType.new("boolean")
+    #
+    #   @example Create a list data type with data type name as Symbol
+    #     Arrow::ListDataType.new(:boolean)
+    #
+    #   @example Create a list data type with data type as Array
+    #     Arrow::ListDataType.new([:time32, :milli])
+    def initialize(arg)
+      data_type = resolve_data_type(arg)
+      if data_type
+        field = Field.new(default_field_name, data_type)
+      else
+        field = resolve_field(arg)
       end
       initialize_raw(field)
     end
+
+    private
+    def resolve_data_type(arg)
+      case arg
+      when DataType, String, Symbol, ::Array
+        DataType.resolve(arg)
+      when Hash
+        return nil if arg[:name]
+        return nil unless arg[:type]
+        DataType.resolve(arg)
+      else
+        nil
+      end
+    end
+
+    def default_field_name
+      "item"
+    end
+
+    def resolve_field(arg)
+      if arg.is_a?(Hash) and arg.key?(:field)
+        description = arg
+        arg = description[:field]
+      end
+      if arg.is_a?(Hash)
+        field_description = arg
+        Field.new(field_description)
+      else
+        arg
+      end
+    end
   end
 end

data/lib/arrow/loader.rb

@@ -32,9 +32,15 @@ module Arrow
     end
 
     def require_libraries
+      require "arrow/column-containable"
+      require "arrow/field-containable"
+      require "arrow/generic-filterable"
+      require "arrow/generic-takeable"
+      require "arrow/record-containable"
+
       require "arrow/array"
       require "arrow/array-builder"
-      require "arrow/binary-array-builder"
+      require "arrow/bigdecimal-extension"
       require "arrow/chunked-array"
       require "arrow/column"
       require "arrow/compression-type"
@@ -53,8 +59,10 @@ module Arrow
       require "arrow/dictionary-data-type"
       require "arrow/field"
       require "arrow/file-output-stream"
+      require "arrow/group"
       require "arrow/list-array-builder"
       require "arrow/list-data-type"
+      require "arrow/null-array"
       require "arrow/null-array-builder"
       require "arrow/path-extension"
       require "arrow/record"

data/lib/arrow/{binary-array-builder.rb → null-array.rb}

@@ -16,21 +16,9 @@
 # under the License.
 
 module Arrow
-  class BinaryArrayBuilder
-    def append_values(values, is_valids=nil)
-      if is_valids
-        is_valids.each_with_index do |is_valid, i|
-          if is_valid
-            append_value(values[i])
-          else
-            append_null
-          end
-        end
-      else
-        values.each do |value|
-          append_value(value)
-        end
-      end
+  class NullArray
+    def get_value(i)
+      nil
     end
   end
 end

data/lib/arrow/record-batch.rb

@@ -15,9 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "arrow/column-containable"
-require "arrow/record-containable"
-
 module Arrow
   class RecordBatch
     include ColumnContainable

data/lib/arrow/schema.rb

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "arrow/field-containable"
-
 module Arrow
   class Schema
     include FieldContainable

data/lib/arrow/struct-data-type.rb

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "arrow/field-containable"
-
 module Arrow
   class StructDataType
     include FieldContainable

data/lib/arrow/table-loader.rb

@@ -41,6 +41,8 @@ module Arrow
             available_formats << match_data.post_match
           end
         end
+        deprecated_formats = ["batch", "stream"]
+        available_formats -= deprecated_formats
         message = "Arrow::Table load format must be one of ["
         message << available_formats.join(", ")
         message << "]: #{format.inspect}"
@@ -119,18 +121,30 @@ module Arrow
       load_raw(input, reader)
     end
 
-    def load_as_batch
+    # @since 1.0.0
+    def load_as_arrow_file
       input = open_input_stream
       reader = RecordBatchFileReader.new(input)
       load_raw(input, reader)
     end
 
-    def load_as_stream
+    # @deprecated Use `format: :arrow_file` instead.
+    def load_as_batch
+      load_as_arrow_file
+    end
+
+    # @since 1.0.0
+    def load_as_arrow_streaming
       input = open_input_stream
       reader = RecordBatchStreamReader.new(input)
       load_raw(input, reader)
     end
 
+    # @deprecated Use `format: :arrow_streaming` instead.
+    def load_as_stream
+      load_as_arrow_streaming
+    end
+
     if Arrow.const_defined?(:ORCFileReader)
       def load_as_orc
         input = open_input_stream
@@ -143,16 +157,25 @@ module Arrow
       end
     end
 
-    def load_as_csv
-      options = @options.dup
+    def csv_load(options)
       options.delete(:format)
       if @input.is_a?(Buffer)
-        CSVLoader.load(@input.data.to_s, options)
+        CSVLoader.load(@input.data.to_s, **options)
       else
-        CSVLoader.load(Pathname.new(@input), options)
+        CSVLoader.load(Pathname.new(@input), **options)
       end
     end
 
+    def load_as_csv
+      csv_load(@options.dup)
+    end
+
+    def load_as_tsv
+      options = @options.dup
+      options[:delimiter] = "\t"
+      csv_load(options.dup)
+    end
+
     def load_as_feather
       input = open_input_stream
       reader = FeatherFileReader.new(input)

data/lib/arrow/table-saver.rb

@@ -42,6 +42,8 @@ module Arrow
             available_formats << match_data.post_match
           end
         end
+        deprecated_formats = ["batch", "stream"]
+        available_formats -= deprecated_formats
         message = "Arrow::Table save format must be one of ["
         message << available_formats.join(", ")
         message << "]: #{format.inspect}"
@@ -110,30 +112,48 @@ module Arrow
     end
 
     def save_as_arrow
-      save_as_batch
+      save_as_arrow_file
     end
 
-    def save_as_batch
+    # @since 1.0.0
+    def save_as_arrow_file
       save_raw(RecordBatchFileWriter)
     end
 
-    def save_as_stream
+    # @deprecated Use `format: :arrow_batch` instead.
+    def save_as_batch
+      save_as_arrow_file
+    end
+
+    # @since 1.0.0
+    def save_as_arrow_streaming
       save_raw(RecordBatchStreamWriter)
     end
 
-    def save_as_csv
+    # @deprecated Use `format: :arrow_streaming` instead.
+    def save_as_stream
+      save_as_arrow_streaming
+    end
+
+    def csv_save(**options)
       open_output_stream do |output|
-        csv = CSV.new(output)
+        csv = CSV.new(output, **options)
         names = @table.schema.fields.collect(&:name)
         csv << names
-        @table.each_record(reuse_record: true) do |record|
-          csv << names.collect do |name|
-            record[name]
-          end
+        @table.raw_records.each do |record|
+          csv << record
         end
       end
     end
 
+    def save_as_csv
+      csv_save
+    end
+
+    def save_as_tsv
+      csv_save(col_sep: "\t")
+    end
+
     def save_as_feather
       open_output_stream do |output|
         FeatherFileWriter.open(output) do |writer|

data/lib/arrow/table.rb

@@ -15,13 +15,11 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "arrow/column-containable"
-require "arrow/group"
-require "arrow/record-containable"
-
 module Arrow
   class Table
     include ColumnContainable
+    include GenericFilterable
+    include GenericTakeable
     include RecordContainable
 
     class << self
@@ -306,13 +304,13 @@ module Arrow
         end
       end
 
-      ranges = []
+      sliced_tables = []
       slicers.each do |slicer|
         slicer = slicer.evaluate if slicer.respond_to?(:evaluate)
         case slicer
         when Integer
           slicer += n_rows if slicer < 0
-          ranges << [slicer, n_rows - 1]
+          sliced_tables << slice_by_range(slicer, n_rows - 1)
         when Range
           original_from = from = slicer.first
           to = slicer.last
@@ -325,17 +323,9 @@ module Arrow
             raise ArgumentError, message
           end
           to += n_rows if to < 0
-          ranges << [from, to]
-        when ::Array
-          boolean_array_to_slice_ranges(slicer, 0, ranges)
-        when ChunkedArray
-          offset = 0
-          slicer.each_chunk do |array|
-            boolean_array_to_slice_ranges(array, offset, ranges)
-            offset += array.length
-          end
-        when BooleanArray
-          boolean_array_to_slice_ranges(slicer, 0, ranges)
+          sliced_tables << slice_by_range(from, to)
+        when ::Array, BooleanArray, ChunkedArray
+          sliced_tables << filter(slicer)
         else
           message = "slicer must be Integer, Range, (from, to), " +
             "Arrow::ChunkedArray of Arrow::BooleanArray, " +
@@ -343,7 +333,11 @@ module Arrow
           raise ArgumentError, message
         end
       end
-      slice_by_ranges(ranges)
+      if sliced_tables.size > 1
+        sliced_tables[0].concatenate(sliced_tables[1..-1])
+      else
+        sliced_tables[0]
+      end
     end
 
     # TODO
@@ -514,38 +508,8 @@ module Arrow
     end
 
     private
-    def boolean_array_to_slice_ranges(array, offset, ranges)
-      in_target = false
-      target_start = nil
-      array.each_with_index do |is_target, i|
-        if is_target
-          unless in_target
-            target_start = offset + i
-            in_target = true
-          end
-        else
-          if in_target
-            ranges << [target_start, offset + i - 1]
-            target_start = nil
-            in_target = false
-          end
-        end
-      end
-      if in_target
-        ranges << [target_start, offset + array.length - 1]
-      end
-    end
-
-    def slice_by_ranges(ranges)
-      sliced_table = []
-      ranges.each do |from, to|
-        sliced_table << slice_raw(from, to - from + 1)
-      end
-      if sliced_table.size > 1
-        sliced_table[0].concatenate(sliced_table[1..-1])
-      else
-        sliced_table[0]
-      end
+    def slice_by_range(from, to)
+      slice_raw(from, to - from + 1)
     end
 
     def ensure_raw_column(name, data)

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "0.15.1"
+  VERSION = "0.16.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -59,5 +59,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency("test-unit")
   spec.add_development_dependency("yard")
 
-  spec.metadata["msys2_mingw_dependencies"] = "arrow"
+  required_msys2_package_version = version_components[0, 3].join(".")
+  spec.metadata["msys2_mingw_dependencies"] =
+    "arrow>=#{required_msys2_package_version}"
 end

data/test/test-array-builder.rb

@@ -60,6 +60,23 @@ class ArrayBuilderTest < Test::Unit::TestCase
                        DateTime.new(2018, 1, 5, 0, 23, 21),
                      ])
       end
+
+      test("list<boolean>s") do
+        assert_build(Arrow::ArrayBuilder,
+                     [
+                       [nil, true, false],
+                       nil,
+                       [false],
+                     ])
+      end
+
+      test("list<string>s") do
+        assert_build(Arrow::ArrayBuilder,
+                     [
+                       ["Hello", "World"],
+                       ["Apache Arrow"],
+                     ])
+      end
     end
 
     sub_test_case("specific builder") do