red-arrow 0.15.0 → 1.0.0

data/lib/arrow/array.rb

@@ -18,20 +18,21 @@
 module Arrow
   class Array
     include Enumerable
+    include GenericFilterable
+    include GenericTakeable
 
     class << self
       def new(*args)
+        _builder_class = builder_class
+        return super if _builder_class.nil?
+        return super unless _builder_class.buildable?(args)
+        _builder_class.build(*args)
+      end
+
+      def builder_class
         builder_class_name = "#{name}Builder"
-        if const_defined?(builder_class_name)
-          builder_class = const_get(builder_class_name)
-          if builder_class.buildable?(args)
-            builder_class.build(*args)
-          else
-            super
-          end
-        else
-          super
-        end
+        return nil unless const_defined?(builder_class_name)
+        const_get(builder_class_name)
       end
     end
 
@@ -82,5 +83,22 @@ module Arrow
     def to_a
       values
     end
+
+    alias_method :is_in_raw, :is_in
+    def is_in(values)
+      case values
+      when ::Array
+        if self.class.builder_class.buildable?([values])
+          values = self.class.new(values)
+        else
+          values = self.class.new(value_data_type, values)
+        end
+        is_in_raw(values)
+      when ChunkedArray
+        is_in_chunked_array(values)
+      else
+        is_in_raw(values)
+      end
+    end
   end
 end

data/lib/arrow/{binary-array-builder.rb → buffer.rb}

@@ -16,21 +16,13 @@
 # under the License.
 
 module Arrow
-  class BinaryArrayBuilder
-    def append_values(values, is_valids=nil)
-      if is_valids
-        is_valids.each_with_index do |is_valid, i|
-          if is_valid
-            append_value(values[i])
-          else
-            append_null
-          end
-        end
-      else
-        values.each do |value|
-          append_value(value)
-        end
-      end
+  class Buffer
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    def initialize(data)
+      @data = data
+      initialize_raw(data)
     end
   end
 end

data/lib/arrow/chunked-array.rb

@@ -18,6 +18,8 @@
 module Arrow
   class ChunkedArray
     include Enumerable
+    include GenericFilterable
+    include GenericTakeable
 
     alias_method :size, :n_rows
     unless method_defined?(:length)

data/lib/arrow/csv-loader.rb

@@ -30,6 +30,9 @@ module Arrow
     def initialize(path_or_data, **options)
       @path_or_data = path_or_data
       @options = options
+      if @options.key?(:delimiter)
+        @options[:col_sep] = @options.delete(:delimiter)
+      end
       @compression = @options.delete(:compression)
     end
 
@@ -93,10 +96,17 @@ module Arrow
       @options.each do |key, value|
         case key
         when :headers
-          if value
-            options.n_header_rows = 1
+          case value
+          when ::Array
+            options.column_names = value
+          when String
+            return nil
           else
-            options.n_header_rows = 0
+            if value
+              options.generate_column_names = false
+            else
+              options.generate_column_names = true
+            end
           end
         when :column_types
           value.each do |name, type|
@@ -106,6 +116,8 @@ module Arrow
           options.add_schema(value)
         when :encoding
           # process encoding on opening input
+        when :col_sep
+          options.delimiter = value
         else
           setter = "#{key}="
           if options.respond_to?(setter)

data/lib/arrow/csv-read-options.rb

@@ -21,5 +21,23 @@ module Arrow
     def add_column_type(name, type)
       add_column_type_raw(name, DataType.resolve(type))
     end
+
+    alias_method :delimiter_raw, :delimiter
+    def delimiter
+      delimiter_raw.chr
+    end
+
+    alias_method :delimiter_raw=, :delimiter=
+    def delimiter=(delimiter)
+      case delimiter
+      when String
+        if delimiter.bytesize != 1
+          message = "delimiter must be 1 byte character: #{delimiter.inspect}"
+          raise ArgumentError, message
+        end
+        delimiter = delimiter.ord
+      end
+      self.delimiter_raw = delimiter
+    end
   end
 end

data/lib/arrow/data-type.rb

@@ -121,6 +121,26 @@ module Arrow
         end
       end
 
+      def sub_types
+        types = {}
+        gtype.children.each do |child|
+          sub_type = child.to_class
+          types[sub_type] = true
+          sub_type.sub_types.each do |sub_sub_type|
+            types[sub_sub_type] = true
+          end
+        end
+        types.keys
+      end
+
+      def try_convert(value)
+        begin
+          resolve(value)
+        rescue ArgumentError
+          nil
+        end
+      end
+
       private
       def resolve_class(data_type)
         components = data_type.to_s.split("_").collect(&:capitalize)
@@ -137,11 +157,24 @@ module Arrow
             available_types << components.collect(&:downcase).join("_").to_sym
           end
           message =
-            "unknown type: #{data_type.inspect}: " +
+            "unknown type: <#{data_type.inspect}>: " +
             "available types: #{available_types.inspect}"
           raise ArgumentError, message
         end
-        Arrow.const_get(data_type_class_name)
+        data_type_class = Arrow.const_get(data_type_class_name)
+        if data_type_class.gtype.abstract?
+          not_abstract_types = data_type_class.sub_types.find_all do |sub_type|
+            not sub_type.gtype.abstract?
+          end
+          not_abstract_types = not_abstract_types.sort_by do |type|
+            type.name
+          end
+          message =
+            "abstract type: <#{data_type.inspect}>: " +
+            "use one of not abstract type: #{not_abstract_types.inspect}"
+          raise ArgumentError, message
+        end
+        data_type_class
       end
     end
 

data/lib/arrow/decimal128-array-builder.rb

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "arrow/bigdecimal-extension"
-
 module Arrow
   class Decimal128ArrayBuilder
     class << self

data/lib/arrow/dictionary-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DictionaryArray
+    def get_value(i)
+      dictionary[indices[i]]
+    end
+  end
+end

data/lib/arrow/field.rb

@@ -59,7 +59,7 @@ module Arrow
     #     There is a shortcut for convenience. If field description
     #     doesn't have `:data_type`, all keys except `:name` are
     #     processes as data type description. For example, the
-    #     following field descrptions are the same:
+    #     following field descriptions are the same:
     #
     #     ```ruby
     #     {name: "visible", data_type: {type: :boolean}}

data/lib/arrow/generic-filterable.rb

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module GenericFilterable
+    class << self
+      def included(base)
+        base.__send__(:alias_method, :filter_raw, :filter)
+        base.__send__(:alias_method, :filter, :filter_generic)
+      end
+    end
+
+    def filter_generic(filter, options=nil)
+      case filter
+      when ::Array
+        filter_raw(BooleanArray.new(filter), options)
+      when ChunkedArray
+        if respond_to?(:filter_chunked_array)
+          filter_chunked_array(filter, options)
+        else
+          # TODO: Implement this in C++
+          filter_raw(filter.pack, options)
+        end
+      else
+        filter_raw(filter, options)
+      end
+    end
+  end
+end

data/lib/arrow/generic-takeable.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module GenericTakeable
+    class << self
+      def included(base)
+        base.__send__(:alias_method, :take_raw, :take)
+        base.__send__(:alias_method, :take, :take_generic)
+      end
+    end
+
+    def take_generic(indices)
+      case indices
+      when ::Array
+        take_raw(IntArrayBuilder.build(indices))
+      when ChunkedArray
+        take_chunked_array(indices)
+      else
+        take_raw(indices)
+      end
+    end
+  end
+end

data/lib/arrow/list-data-type.rb

@@ -53,16 +53,66 @@ module Arrow
     #
     #   @example Create a list data type with field description
     #     Arrow::ListDataType.new(field: {name: "visible", type: :boolean})
-    def initialize(field)
-      if field.is_a?(Hash) and field.key?(:field)
-        description = field
-        field = description[:field]
-      end
-      if field.is_a?(Hash)
-        field_description = field
-        field = Field.new(field_description)
+    #
+    # @overload initialize(data_type)
+    #
+    #   @param data_type [Arrow::DataType, String, Symbol,
+    #     ::Array<String>, ::Array<Symbol>, Hash] The element data
+    #     type of the list data type. A field is created with the
+    #     default name `"item"` from the data type automatically.
+    #
+    #     See {Arrow::DataType.resolve} how to specify data type.
+    #
+    #   @example Create a list data type with {Arrow::DataType}
+    #     Arrow::ListDataType.new(Arrow::BooleanDataType.new)
+    #
+    #   @example Create a list data type with data type name as String
+    #     Arrow::ListDataType.new("boolean")
+    #
+    #   @example Create a list data type with data type name as Symbol
+    #     Arrow::ListDataType.new(:boolean)
+    #
+    #   @example Create a list data type with data type as Array
+    #     Arrow::ListDataType.new([:time32, :milli])
+    def initialize(arg)
+      data_type = resolve_data_type(arg)
+      if data_type
+        field = Field.new(default_field_name, data_type)
+      else
+        field = resolve_field(arg)
       end
       initialize_raw(field)
     end
+
+    private
+    def resolve_data_type(arg)
+      case arg
+      when DataType, String, Symbol, ::Array
+        DataType.resolve(arg)
+      when Hash
+        return nil if arg[:name]
+        return nil unless arg[:type]
+        DataType.resolve(arg)
+      else
+        nil
+      end
+    end
+
+    def default_field_name
+      "item"
+    end
+
+    def resolve_field(arg)
+      if arg.is_a?(Hash) and arg.key?(:field)
+        description = arg
+        arg = description[:field]
+      end
+      if arg.is_a?(Hash)
+        field_description = arg
+        Field.new(field_description)
+      else
+        arg
+      end
+    end
   end
 end

data/lib/arrow/loader.rb

@@ -32,9 +32,16 @@ module Arrow
     end
 
     def require_libraries
+      require "arrow/column-containable"
+      require "arrow/field-containable"
+      require "arrow/generic-filterable"
+      require "arrow/generic-takeable"
+      require "arrow/record-containable"
+
       require "arrow/array"
       require "arrow/array-builder"
-      require "arrow/binary-array-builder"
+      require "arrow/bigdecimal-extension"
+      require "arrow/buffer"
       require "arrow/chunked-array"
       require "arrow/column"
       require "arrow/compression-type"
@@ -50,17 +57,21 @@ module Arrow
       require "arrow/decimal128-array-builder"
       require "arrow/decimal128-data-type"
       require "arrow/dense-union-data-type"
+      require "arrow/dictionary-array"
       require "arrow/dictionary-data-type"
       require "arrow/field"
       require "arrow/file-output-stream"
+      require "arrow/group"
       require "arrow/list-array-builder"
       require "arrow/list-data-type"
+      require "arrow/null-array"
       require "arrow/null-array-builder"
       require "arrow/path-extension"
       require "arrow/record"
       require "arrow/record-batch"
       require "arrow/record-batch-builder"
       require "arrow/record-batch-file-reader"
+      require "arrow/record-batch-iterator"
       require "arrow/record-batch-stream-reader"
       require "arrow/rolling-window"
       require "arrow/schema"