red-arrow 0.15.1 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04ae0088ccf17ff76c09ad884e9c4ce6c9a9c637c8f724964c24a379e37d652e
-  data.tar.gz: a3387f67b30dfd07963730ff79c694d703a08f872fdb7990509d019b460e20b9
+  metadata.gz: 64b14ef4120f4ab290e8161020902ec2a22631c519d5a133a63ce383610e8545
+  data.tar.gz: 2f5850520e2dc69568a454cee0d4246909d52f1d49851221b1b9efd3149bc15c
 SHA512:
-  metadata.gz: dc03abf2065a62ffeb9061f64d9e6074dd90dee464bddc4a06773635a727abc025f6b249dad19bcdfa207308ecf24e7a5f8b7e18ec127c897f63d5916a20aa31
-  data.tar.gz: 7dc98d5e71d23595baffda2b167494677a2d7d2269b010bea23555a6442d848248068f1e87ae3fa79c9fb61957243f038438ae3fa429f4daae3ead429c9066a1
+  metadata.gz: 0e19a4da6182437a51f9dad6212436e70b00881674ee6cd29e2a40910fe711fdef4076c81d2778f3ff9e9dd3f45b573c43e90378244cf08d7550b504ad8b53af
+  data.tar.gz: 547eb8b31fd59d9c1d5fc1163bb25da70154b797d103d3a01f6fda70dddc0b3c2cdb5391b9006c4510be37b8b75988195f809aaead8c91e7fb68f762cc5313de

data/ext/arrow/converters.hpp

@@ -504,14 +504,14 @@ namespace red_arrow {
     uint8_t compute_child_index(const arrow::UnionArray& array,
                                 arrow::UnionType* type,
                                 const char* tag) {
-      const auto type_id = array.raw_type_ids()[index_];
-      const auto& type_codes = type->type_codes();
-      for (uint8_t i = 0; i < type_codes.size(); ++i) {
-        if (type_codes[i] == type_id) {
-          return i;
+      const auto type_code = array.raw_type_codes()[index_];
+      if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
+        const auto child_id = type->child_ids()[type_code];
+        if (child_id >= 0) {
+          return child_id;
         }
       }
-      check_status(arrow::Status::Invalid("Unknown type ID: ", type_id),
+      check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
                    tag);
       return 0;
     }

data/lib/arrow/array-builder.rb

@@ -26,60 +26,13 @@ module Arrow
           return builder.build(values)
         end
 
-        builder_class = nil
-        builder_class_arguments = []
+        builder_info = nil
         values.each do |value|
-          case value
-          when nil
-            # Ignore
-          when true, false
-            return BooleanArray.new(values)
-          when String
-            return StringArray.new(values)
-          when Float
-            return DoubleArray.new(values)
-          when Integer
-            if value < 0
-              builder = IntArrayBuilder.new
-              return builder.build(values)
-            else
-              builder_class = UIntArrayBuilder
-              builder_class_arguments = []
-            end
-          when Time
-            data_type = value.data_type
-            case data_type.unit
-            when TimeUnit::SECOND
-              if builder.nil?
-                builder = Time32ArrayBuilder
-                builder_class_arguments = [data_type]
-              end
-            when TimeUnit::MILLI
-              if builder != Time64ArrayBuilder
-                builder = Time32ArrayBuilder
-                builder_class_arguments = [data_type]
-              end
-            when TimeUnit::MICRO
-              builder = Time64ArrayBuilder
-              builder_class_arguments = [data_type]
-            when TimeUnit::NANO
-              builder = Time64ArrayBuilder.new(data_type)
-              return builder.build(values)
-            end
-          when ::Time
-            data_type = TimestampDataType.new(:nano)
-            builder = TimestampArrayBuilder.new(data_type)
-            return builder.build(values)
-          when DateTime
-            return Date64Array.new(values)
-          when Date
-            return Date32Array.new(values)
-          else
-            return StringArray.new(values)
-          end
+          builder_info = detect_builder_info(value, builder_info)
+          break if builder_info and builder_info[:detected]
         end
-        if builder_class
-          builder = builder_class.new(*builder_class_arguments)
+        if builder_info
+          builder = builder_info[:builder]
           builder.build(values)
         else
           Arrow::StringArray.new(values)
@@ -89,6 +42,102 @@ module Arrow
       def buildable?(args)
         args.size == method(:build).arity
       end
+
+      private
+      def detect_builder_info(value, builder_info)
+        case value
+        when nil
+          builder_info
+        when true, false
+          {
+            builder: BooleanArrayBuilder.new,
+            detected: true,
+          }
+        when String
+          {
+            builder: StringArrayBuilder.new,
+            detected: true,
+          }
+        when Float
+          {
+            builder: DoubleArrayBuilder.new,
+            detected: true,
+          }
+        when Integer
+          if value < 0
+            {
+              builder: IntArrayBuilder.new,
+              detected: true,
+            }
+          else
+            {
+              builder: UIntArrayBuilder.new,
+            }
+          end
+        when Time
+          data_type = value.data_type
+          case data_type.unit
+          when TimeUnit::SECOND
+            builder_info || {
+              builder: Time32ArrayBuilder.new(data_type)
+            }
+          when TimeUnit::MILLI
+            if builder_info and builder_info[:builder].is_a?(Time64ArrayBuilder)
+              builder_info
+            else
+              {
+                builder: Time32ArrayBuilder.new(data_type),
+              }
+            end
+          when TimeUnit::MICRO
+            {
+              builder: Time64ArrayBuilder.new(data_type),
+            }
+          when TimeUnit::NANO
+            {
+              builder: Time64ArrayBuilder.new(data_type),
+              detected: true
+            }
+          end
+        when ::Time
+          data_type = TimestampDataType.new(:nano)
+          {
+            builder: TimestampArrayBuilder.new(data_type),
+            detected: true,
+          }
+        when DateTime
+          {
+            builder: Date64ArrayBuilder.new,
+            detected: true,
+          }
+        when Date
+          {
+            builder: Date32ArrayBuilder.new,
+            detected: true,
+          }
+        when ::Array
+          sub_builder_info = nil
+          value.each do |sub_value|
+            sub_builder_info = detect_builder_info(sub_value, sub_builder_info)
+            break if sub_builder_info and sub_builder_info[:detected]
+          end
+          if sub_builder_info and sub_builder_info[:detected]
+            sub_value_data_type = sub_builder_info[:builder].value_data_type
+            field = Field.new("item", sub_value_data_type)
+            {
+              builder: ListArrayBuilder.new(ListDataType.new(field)),
+              detected: true,
+            }
+          else
+            builder_info
+          end
+        else
+          {
+            builder: StringArrayBuilder.new,
+            detected: true,
+          }
+        end
+      end
     end
 
     def build(values)

data/lib/arrow/array.rb

@@ -18,20 +18,21 @@
 module Arrow
   class Array
     include Enumerable
+    include GenericFilterable
+    include GenericTakeable
 
     class << self
       def new(*args)
+        _builder_class = builder_class
+        return super if _builder_class.nil?
+        return super unless _builder_class.buildable?(args)
+        _builder_class.build(*args)
+      end
+
+      def builder_class
         builder_class_name = "#{name}Builder"
-        if const_defined?(builder_class_name)
-          builder_class = const_get(builder_class_name)
-          if builder_class.buildable?(args)
-            builder_class.build(*args)
-          else
-            super
-          end
-        else
-          super
-        end
+        return nil unless const_defined?(builder_class_name)
+        const_get(builder_class_name)
       end
     end
 
@@ -82,5 +83,22 @@ module Arrow
     def to_a
       values
     end
+
+    alias_method :is_in_raw, :is_in
+    def is_in(values)
+      case values
+      when ::Array
+        if self.class.builder_class.buildable?([values])
+          values = self.class.new(values)
+        else
+          values = self.class.new(value_data_type, values)
+        end
+        is_in_raw(values)
+      when ChunkedArray
+        is_in_chunked_array(values)
+      else
+        is_in_raw(values)
+      end
+    end
   end
 end

data/lib/arrow/chunked-array.rb

@@ -18,6 +18,8 @@
 module Arrow
   class ChunkedArray
     include Enumerable
+    include GenericFilterable
+    include GenericTakeable
 
     alias_method :size, :n_rows
     unless method_defined?(:length)

data/lib/arrow/csv-loader.rb

@@ -30,6 +30,9 @@ module Arrow
     def initialize(path_or_data, **options)
       @path_or_data = path_or_data
       @options = options
+      if @options.key?(:delimiter)
+        @options[:col_sep] = @options.delete(:delimiter)
+      end
       @compression = @options.delete(:compression)
     end
 
@@ -113,6 +116,8 @@ module Arrow
           options.add_schema(value)
         when :encoding
           # process encoding on opening input
+        when :col_sep
+          options.delimiter = value
         else
           setter = "#{key}="
           if options.respond_to?(setter)

data/lib/arrow/csv-read-options.rb

@@ -21,5 +21,23 @@ module Arrow
     def add_column_type(name, type)
       add_column_type_raw(name, DataType.resolve(type))
     end
+
+    alias_method :delimiter_raw, :delimiter
+    def delimiter
+      delimiter_raw.chr
+    end
+
+    alias_method :delimiter_raw=, :delimiter=
+    def delimiter=(delimiter)
+      case delimiter
+      when String
+        if delimiter.bytesize != 1
+          message = "delimiter must be 1 byte character: #{delimiter.inspect}"
+          raise ArgumentError, message
+        end
+        delimiter = delimiter.ord
+      end
+      self.delimiter_raw = delimiter
+    end
   end
 end

data/lib/arrow/data-type.rb

@@ -121,6 +121,26 @@ module Arrow
         end
       end
 
+      def sub_types
+        types = {}
+        gtype.children.each do |child|
+          sub_type = child.to_class
+          types[sub_type] = true
+          sub_type.sub_types.each do |sub_sub_type|
+            types[sub_sub_type] = true
+          end
+        end
+        types.keys
+      end
+
+      def try_convert(value)
+        begin
+          resolve(value)
+        rescue ArgumentError
+          nil
+        end
+      end
+
       private
       def resolve_class(data_type)
         components = data_type.to_s.split("_").collect(&:capitalize)
@@ -137,11 +157,24 @@ module Arrow
             available_types << components.collect(&:downcase).join("_").to_sym
           end
           message =
-            "unknown type: #{data_type.inspect}: " +
+            "unknown type: <#{data_type.inspect}>: " +
             "available types: #{available_types.inspect}"
           raise ArgumentError, message
         end
-        Arrow.const_get(data_type_class_name)
+        data_type_class = Arrow.const_get(data_type_class_name)
+        if data_type_class.gtype.abstract?
+          not_abstract_types = data_type_class.sub_types.find_all do |sub_type|
+            not sub_type.gtype.abstract?
+          end
+          not_abstract_types = not_abstract_types.sort_by do |type|
+            type.name
+          end
+          message =
+            "abstract type: <#{data_type.inspect}>: " +
+            "use one of not abstract type: #{not_abstract_types.inspect}"
+          raise ArgumentError, message
+        end
+        data_type_class
       end
     end
 

data/lib/arrow/decimal128-array-builder.rb

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "arrow/bigdecimal-extension"
-
 module Arrow
   class Decimal128ArrayBuilder
     class << self

data/lib/arrow/field.rb

@@ -59,7 +59,7 @@ module Arrow
     #     There is a shortcut for convenience. If field description
     #     doesn't have `:data_type`, all keys except `:name` are
     #     processes as data type description. For example, the
-    #     following field descrptions are the same:
+    #     following field descriptions are the same:
     #
     #     ```ruby
     #     {name: "visible", data_type: {type: :boolean}}

data/lib/arrow/generic-filterable.rb

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module GenericFilterable
+    class << self
+      def included(base)
+        base.alias_method :filter_raw, :filter
+        base.alias_method :filter, :filter_generic
+      end
+    end
+
+    def filter_generic(filter)
+      case filter
+      when ::Array
+        filter_raw(BooleanArray.new(filter))
+      when ChunkedArray
+        if respond_to?(:filter_chunked_array)
+          filter_chunked_array(filter)
+        else
+          # TODO: Implement this in C++
+          filter_raw(filter.pack)
+        end
+      else
+        filter_raw(filter)
+      end
+    end
+  end
+end

data/lib/arrow/generic-takeable.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module GenericTakeable
+    class << self
+      def included(base)
+        base.alias_method :take_raw, :take
+        base.alias_method :take, :take_generic
+      end
+    end
+
+    def take_generic(indices)
+      case indices
+      when ::Array
+        take_raw(IntArrayBuilder.build(indices))
+      when ChunkedArray
+        take_chunked_array(indices)
+      else
+        take_raw(indices)
+      end
+    end
+  end
+end