red-arrow 10.0.0 → 16.0.0

data/lib/arrow/dense-union-array.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DenseUnionArray
+    def get_value(i)
+      child_id = get_child_id(i)
+      field = get_field(child_id)
+      field[get_value_offset(i)]
+    end
+  end
+end

data/lib/arrow/expression.rb

@@ -31,10 +31,14 @@ module Arrow
           else
             return nil
           end
+          options = nil
           if arguments.last.is_a?(FunctionOptions)
             options = arguments.pop
-          else
-            options = nil
+          elsif arguments.last.is_a?(Hash)
+            function = Function.find(function_name)
+            if function
+              options = function.resolve_options(arguments.pop)
+            end
           end
           CallExpression.new(function_name, arguments, options)
         else

data/lib/arrow/function.rb

@@ -24,7 +24,6 @@ module Arrow
     end
     alias_method :call, :execute
 
-    private
     def resolve_options(options)
       return nil if options.nil?
       return options if options.is_a?(FunctionOptions)

data/lib/arrow/half-float-array-builder.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloatArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      case value
+      when Float
+        HalfFloat.new(value).to_uint16
+      when HalfFloat
+        value.to_uint16
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/half-float-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloatArray
+    def get_value(i)
+      HalfFloat.new(get_raw_value(i)).to_f
+    end
+  end
+end

data/lib/arrow/half-float.rb

@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloat
+    MAX = 65504
+    MIN = -65504
+    EXPONENT_N_BITS = 5
+    EXPONENT_MASK = (2 ** EXPONENT_N_BITS) - 1
+    EXPONENT_BIAS = 15
+    FRACTION_N_BITS = 10
+    FRACTION_MASK = (2 ** FRACTION_N_BITS) - 1
+    FRACTION_DENOMINATOR = 2.0 ** FRACTION_N_BITS
+
+    attr_reader :sign
+    attr_reader :exponent
+    attr_reader :fraction
+    def initialize(*args)
+      n_args = args.size
+      case n_args
+      when 1
+        if args[0].is_a?(Float)
+          @sign, @exponent, @fraction = deconstruct_float(args[0])
+        else
+          @sign, @exponent, @fraction = deconstruct_uint16(args[0])
+        end
+      when 3
+        @sign, @exponent, @fraction = *args
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 1 or 3)"
+        raise ArgumentError, message
+      end
+    end
+
+    def to_f
+      if @exponent == EXPONENT_MASK
+        if @sign.zero?
+          Float::INFINITY
+        else
+          -Float::INFINITY
+        end
+      else
+        if @exponent.zero?
+          implicit_fraction = 0
+        else
+          implicit_fraction = 1
+        end
+        ((-1) ** @sign) *
+          (2 ** (@exponent - EXPONENT_BIAS)) *
+          (implicit_fraction + @fraction / FRACTION_DENOMINATOR)
+      end
+    end
+
+    def to_uint16
+      (@sign << (EXPONENT_N_BITS + FRACTION_N_BITS)) ^
+        (@exponent << FRACTION_N_BITS) ^
+        @fraction
+    end
+
+    def pack
+      [to_uint16].pack("S")
+    end
+
+    private
+    def deconstruct_float(float)
+      if float > MAX
+        float = Float::INFINITY
+      elsif float < MIN
+        float = -Float::INFINITY
+      end
+      is_infinite = float.infinite?
+      if is_infinite
+        sign = (is_infinite == 1) ? 0 : 1
+        exponent = EXPONENT_MASK
+        fraction = 0
+      elsif float.zero?
+        sign = 0
+        exponent = 0
+        fraction = 0
+      else
+        sign = (float.positive? ? 0 : 1)
+        float_abs = float.abs
+        1.upto(EXPONENT_MASK) do |e|
+          next_exponent_value = 2 ** (e + 1 - EXPONENT_BIAS)
+          next if float_abs > next_exponent_value
+          exponent = e
+          exponent_value = 2 ** (e - EXPONENT_BIAS)
+          fraction =
+            ((float_abs / exponent_value - 1) * FRACTION_DENOMINATOR).round
+          break
+        end
+      end
+      [sign, exponent, fraction]
+    end
+
+    def deconstruct_uint16(uint16)
+      # | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
+      sign = (uint16 >> (EXPONENT_N_BITS + FRACTION_N_BITS))
+      exponent = ((uint16 >> FRACTION_N_BITS) & EXPONENT_MASK)
+      fraction = (uint16 & FRACTION_MASK)
+      [sign, exponent, fraction]
+    end
+  end
+end

data/lib/arrow/input-referable.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module InputReferable
+    def refer_input(input)
+      @input = input
+    end
+
+    def share_input(other)
+      return unless defined?(@input)
+      other.refer_input(@input)
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -39,6 +39,7 @@ module Arrow
       require "arrow/field-containable"
       require "arrow/generic-filterable"
       require "arrow/generic-takeable"
+      require "arrow/input-referable"
       require "arrow/record-containable"
       require "arrow/symbol-values-appendable"
 
@@ -69,6 +70,8 @@ module Arrow
       require "arrow/decimal256-array"
       require "arrow/decimal256-array-builder"
       require "arrow/decimal256-data-type"
+      require "arrow/dense-union-array"
+      require "arrow/dense-union-array-builder"
       require "arrow/dense-union-data-type"
       require "arrow/dictionary-array"
       require "arrow/dictionary-data-type"
@@ -81,6 +84,9 @@ module Arrow
       require "arrow/fixed-size-binary-array-builder"
       require "arrow/function"
       require "arrow/group"
+      require "arrow/half-float"
+      require "arrow/half-float-array"
+      require "arrow/half-float-array-builder"
       require "arrow/list-array-builder"
       require "arrow/list-data-type"
       require "arrow/map-array"
@@ -105,6 +111,8 @@ module Arrow
       require "arrow/sort-key"
       require "arrow/sort-options"
       require "arrow/source-node-options"
+      require "arrow/sparse-union-array"
+      require "arrow/sparse-union-array-builder"
       require "arrow/sparse-union-data-type"
       require "arrow/string-dictionary-array-builder"
       require "arrow/string-array-builder"
@@ -130,6 +138,8 @@ module Arrow
       require "arrow/timestamp-array"
       require "arrow/timestamp-array-builder"
       require "arrow/timestamp-data-type"
+      require "arrow/timestamp-parser"
+      require "arrow/union-array-builder"
       require "arrow/writable"
     end
 
@@ -196,6 +206,7 @@ module Arrow
            "Arrow::Date64Array",
            "Arrow::Decimal128Array",
            "Arrow::Decimal256Array",
+           "Arrow::HalfFloatArray",
            "Arrow::Time32Array",
            "Arrow::Time64Array",
            "Arrow::TimestampArray"

data/lib/arrow/raw-table-converter.rb

@@ -35,14 +35,16 @@ module Arrow
         fields = []
         @values = []
         @raw_table.each do |name, array|
-          if array.respond_to?(:to_arrow_array)
-            array = array.to_arrow_array
+          if array.respond_to?(:to_arrow_chunked_array)
+            chunked_array = array.to_arrow_chunked_array
+          elsif array.respond_to?(:to_arrow_array)
+            chunked_array = ChunkedArray.new([array.to_arrow_array])
           else
             array = array.to_ary if array.respond_to?(:to_ary)
-            array = ArrayBuilder.build(array)
+            chunked_array = ChunkedArray.new([ArrayBuilder.build(array)])
           end
-          fields << Field.new(name.to_s, array.value_data_type)
-          @values << array
+          fields << Field.new(name.to_s, chunked_array.value_data_type)
+          @values << chunked_array
         end
         @schema = Schema.new(fields)
       end

data/lib/arrow/record-batch-file-reader.rb

@@ -20,6 +20,8 @@ module Arrow
     include Enumerable
 
     def each
+      return to_enum(__method__) {n_record_batches} unless block_given?
+
       n_record_batches.times do |i|
         yield(get_record_batch(i))
       end

data/lib/arrow/record-batch-stream-reader.rb

@@ -20,6 +20,8 @@ module Arrow
     include Enumerable
 
     def each
+      return to_enum(__method__) unless block_given?
+
       loop do
         record_batch = next_record_batch
         break if record_batch.nil?

data/lib/arrow/record-batch.rb

@@ -19,9 +19,11 @@ require "arrow/raw-table-converter"
 
 module Arrow
   class RecordBatch
+    include Enumerable
+
     include ColumnContainable
+    include InputReferable
     include RecordContainable
-    include Enumerable
 
     class << self
       def new(*args)
@@ -56,7 +58,9 @@ module Arrow
     #
     # @since 0.12.0
     def to_table
-      Table.new(schema, [self])
+      table = Table.new(schema, [self])
+      share_input(table)
+      table
     end
 
     def respond_to_missing?(name, include_private)

data/lib/arrow/scalar.rb

@@ -17,6 +17,73 @@
 
 module Arrow
   class Scalar
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when self
+          value
+        when true, false
+          BooleanScalar.new(value)
+        when Symbol, String
+          StringScalar.new(value.to_s)
+        when Integer
+          Int64Scalar.new(value)
+        when Float
+          DoubleScalar.new(value)
+        else
+          nil
+        end
+      end
+
+      # Ensure returning suitable {Arrow::Scalar}.
+      #
+      # @overload resolve(scalar)
+      #
+      #   Returns the given scalar itself. This is convenient to
+      #   use this method as {Arrow::Scalar} converter.
+      #
+      #   @param scalar [Arrow::Scalar] The scalar.
+      #
+      #   @return [Arrow::Scalar] The given scalar itself.
+      #
+      # @overload resolve(value)
+      #
+      #   Creates a suitable scalar from the given value. For example,
+      #   you can create {Arrow::BooleanScalar} from `true`.
+      #
+      #   @param value [Object] The value.
+      #
+      #   @return [Arrow::Scalar] A suitable {Arrow::Scalar} for `value`.
+      #
+      # @overload resolve(value, data_type)
+      #
+      #   Creates a scalar of `data_type.scalar_class` from the given
+      #   value. For example, you can create {Arrow::Int32Scalar} from
+      #   `29` and {Arrow::Int32DataType}.
+      #
+      #   @param value [Object] The value.
+      #
+      #   @param data_type [Arrow::DataType] The {Arrow::DataType} to
+      #     decide the returned scalar class.
+      #
+      #   @return [Arrow::Scalar] A suitable {Arrow::Scalar} for `value`.
+      #
+      # @since 12.0.0
+      def resolve(value, data_type=nil)
+        return try_convert(value) if data_type.nil?
+
+        data_type = DataType.resolve(data_type)
+        scalar_class = data_type.scalar_class
+        case value
+        when Scalar
+          return value if value.class == scalar_class
+          value = value.value
+        end
+        scalar_class.new(value)
+      end
+    end
+
     # @param other [Arrow::Scalar] The scalar to be compared.
     # @param options [Arrow::EqualOptions, Hash] (nil)
     #   The options to custom how to compare.

data/lib/arrow/slicer.rb

@@ -162,6 +162,40 @@ module Arrow
       def reject(&block)
         RejectCondition.new(@column, block)
       end
+
+      def end_with?(substring, ignore_case: false)
+        MatchSubstringFamilyCondition.new("ends_with",
+                                          @column, substring, ignore_case)
+      end
+
+      def match_like?(pattern, ignore_case: false)
+        MatchSubstringFamilyCondition.new("match_like",
+                                          @column, pattern, ignore_case)
+      end
+
+      def match_substring?(pattern, ignore_case: nil)
+        case pattern
+        when String
+          ignore_case = false if ignore_case.nil?
+          MatchSubstringFamilyCondition.new("match_substring",
+                                            @column, pattern, ignore_case)
+        when Regexp
+          ignore_case = pattern.casefold? if ignore_case.nil?
+          MatchSubstringFamilyCondition.new("match_substring_regex",
+                                            @column,
+                                            pattern.source,
+                                            ignore_case)
+        else
+          message =
+             "pattern must be either String or Regexp: #{pattern.inspect}"
+          raise ArgumentError, message
+        end
+      end
+
+      def start_with?(substring, ignore_case: false)
+        MatchSubstringFamilyCondition.new("starts_with",
+                                          @column, substring, ignore_case)
+      end
     end
 
     class NotColumnCondition < Condition
@@ -351,5 +385,32 @@ module Arrow
         BooleanArray.new(raw_array)
       end
     end
+
+    class MatchSubstringFamilyCondition < Condition
+      def initialize(function, column, pattern, ignore_case, invert: false)
+        @function = function
+        @column = column
+        @options = MatchSubstringOptions.new
+        @options.pattern = pattern
+        @options.ignore_case = ignore_case
+        @invert = invert
+      end
+
+      def !@
+        MatchSubstringFamilyCondition.new(@function,
+                                          @column,
+                                          @options.pattern,
+                                          @options.ignore_case?,
+                                          invert: !@invert)
+      end
+
+      def evaluate
+        datum = Function.find(@function).execute([@column.data], @options)
+        if @invert
+          datum = Function.find("invert").execute([datum])
+        end
+        datum.value
+      end
+    end
   end
 end

data/lib/arrow/sort-key.rb

@@ -79,9 +79,9 @@ module Arrow
     #     target and corresponding order is used. `"+"` uses ascending
     #     order and `"-"` uses ascending order.
     #
-    #     If `target` is not a String nor `target` doesn't start with the
-    #     leading order mark, sort column target is `target` as-is and
-    #     ascending order is used.
+    #     If `target` is either not a String or `target` doesn't start
+    #     with the leading order mark, sort column is `target` as-is
+    #     and ascending order is used.
     #
     #   @example String without the leading order mark
     #     key = Arrow::SortKey.new("count")

data/lib/arrow/sparse-union-array-builder.rb

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class SparseUnionArrayBuilder
+    alias_method :append_value_raw, :append_value
+
+    # @overload append_value
+    #
+    #   Starts appending an union record. You need to append values of
+    #   fields.
+    #
+    # @overload append_value(value)
+    #
+    #   Appends an union record including values of fields.
+    #
+    #   @param value [nil, Hash] The union record value.
+    #
+    #     If this is `nil`, the union record is null.
+    #
+    #     If this is `Hash`, it's  values of fields.
+    #
+    # @since 12.0.0
+    def append_value(value)
+      if value.nil?
+        append_null
+      else
+        key = value.keys[0]
+        child_info = child_infos[key]
+        append_value_raw(child_info[:id])
+        child_infos.each do |child_key, child_info|
+          builder = child_info[:builder]
+          if child_key == key
+            builder.append(value.values[0])
+          else
+            builder.append_null
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow/sparse-union-array.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class SparseUnionArray
+    def get_value(i)
+      child_id = get_child_id(i)
+      field = get_field(child_id)
+      field[i]
+    end
+  end
+end

data/lib/arrow/struct-array-builder.rb

@@ -110,11 +110,6 @@ module Arrow
       end
     end
 
-    alias_method :append_null_raw, :append_null
-    def append_null
-      append_null_raw
-    end
-
     # @since 0.12.0
     def append(*values)
       if values.empty?

data/lib/arrow/table-loader.rb

@@ -161,7 +161,7 @@ module Arrow
         record_batches << record_batch
       end
       table = Table.new(schema, record_batches)
-      table.instance_variable_set(:@input, input)
+      table.refer_input(input)
       table
     end
 
@@ -211,7 +211,7 @@ module Arrow
           field_indexes = @options[:field_indexes]
           reader.set_field_indexes(field_indexes) if field_indexes
           table = reader.read_stripes
-          table.instance_variable_set(:@input, input)
+          table.refer_input(input)
           table
         end
       end
@@ -245,16 +245,22 @@ module Arrow
       open_input_stream do |input|
         reader = FeatherFileReader.new(input)
         table = reader.read
-        table.instance_variable_set(:@input, input)
+        table.refer_input(input)
         table
       end
     end
 
     def load_as_json
       open_input_stream do |input|
-        reader = JSONReader.new(input)
+        options = JSONReadOptions.new
+        @options.each do |key, value|
+          next if value.nil?
+          setter = :"#{key}="
+          options.__send__(setter, value) if options.respond_to?(setter)
+        end
+        reader = JSONReader.new(input, options)
         table = reader.read
-        table.instance_variable_set(:@input, input)
+        table.refer_input(input)
         table
       end
     end