red-arrow 8.0.0 → 24.0.0

data/lib/arrow/fixed-size-list-data-type.rb

@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class FixedSizeListDataType
+    include ListFieldResolvable
+
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    # Creates a new {Arrow::FixedSizeListDataType}.
+    #
+    # @overload initialize(field, size)
+    #
+    #   @param field [Arrow::Field, Hash] The field of the fixed size
+    #     list data type. You can also specify field description by
+    #     `Hash`.
+    #
+    #     See {Arrow::Field.new} how to specify field description.
+    #
+    #   @param size [Integer] The number of values in each element.
+    #
+    #   @example Create a fixed size list data type with {Arrow::Field}
+    #     visible_field = Arrow::Field.new("visible", :boolean)
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new(visible_field, size)
+    #
+    #   @example Create a list data type with field description
+    #     description = {name: "visible", type: :boolean}
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new(description, size)
+    #
+    # @overload initialize(description)
+    #
+    #   @param description [Hash] The description of the fixed size
+    #     list data type. It must have `:field` value and `:size`
+    #     value.
+    #
+    #   @option description [Arrow::Field, Hash] :field The field of
+    #     the list data type. You can also specify field description
+    #     by `Hash`.
+    #
+    #     See {Arrow::Field.new} how to specify field description.
+    #
+    #   @option description [Integer] :size The number of values of
+    #     each element of the fixed size list data type.
+    #
+    #   @example Create a fixed size list data type with {Arrow::Field}
+    #     visible_field = Arrow::Field.new("visible", :boolean)
+    #     Arrow::FixedSizeListDataType.new(field: visible_field, size: 2)
+    #
+    #   @example Create a fixed size list data type with field description
+    #     Arrow::FixedSizeListDataType.new(field: {
+    #                                        name: "visible",
+    #                                        type: :boolean,
+    #                                      },
+    #                                      size: 2)
+    #
+    # @overload initialize(data_type, size)
+    #
+    #   @param data_type [Arrow::DataType, String, Symbol,
+    #     ::Array<String>, ::Array<Symbol>, Hash] The element data
+    #     type of the fixed size list data type. A field is created
+    #     with the default name `"item"` from the data type
+    #     automatically.
+    #
+    #     See {Arrow::DataType.resolve} how to specify data type.
+    #
+    #   @param size [Integer] The number of values in each
+    #     element.
+    #
+    #   @example Create a fixed size list data type with {Arrow::DataType}
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new(Arrow::BooleanDataType.new,
+    #                                      size)
+    #
+    #   @example Create a fixed size list data type with data type name as String
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new("boolean", size)
+    #
+    #   @example Create a fixed size list data type with data type name as Symbol
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new(:boolean, size)
+    #
+    #   @example Create a fixed size list data type with data type as Array
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new([:time32, :milli], size)
+    def initialize(*args)
+      n_args = args.size
+      case n_args
+      when 1
+        description = args[0]
+        size = description.delete(:size)
+        initialize_raw(resolve_field(description), size)
+      when 2
+        field, size = args
+        initialize_raw(resolve_field(field), size)
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 1..2)"
+        raise ArgumentError, message
+      end
+    end
+  end
+end

data/lib/arrow/function.rb

@@ -24,7 +24,6 @@ module Arrow
     end
     alias_method :call, :execute
 
-    private
     def resolve_options(options)
       return nil if options.nil?
       return options if options.is_a?(FunctionOptions)

data/lib/arrow/half-float-array-builder.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloatArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      case value
+      when Float
+        HalfFloat.new(value).to_uint16
+      when HalfFloat
+        value.to_uint16
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/half-float-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloatArray
+    def get_value(i)
+      HalfFloat.new(get_raw_value(i)).to_f
+    end
+  end
+end

data/lib/arrow/half-float.rb

@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloat
+    MAX = 65504
+    MIN = -65504
+    EXPONENT_N_BITS = 5
+    EXPONENT_MASK = (2 ** EXPONENT_N_BITS) - 1
+    EXPONENT_BIAS = 15
+    FRACTION_N_BITS = 10
+    FRACTION_MASK = (2 ** FRACTION_N_BITS) - 1
+    FRACTION_DENOMINATOR = 2.0 ** FRACTION_N_BITS
+
+    attr_reader :sign
+    attr_reader :exponent
+    attr_reader :fraction
+    def initialize(*args)
+      n_args = args.size
+      case n_args
+      when 1
+        if args[0].is_a?(Float)
+          @sign, @exponent, @fraction = deconstruct_float(args[0])
+        else
+          @sign, @exponent, @fraction = deconstruct_uint16(args[0])
+        end
+      when 3
+        @sign, @exponent, @fraction = *args
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 1 or 3)"
+        raise ArgumentError, message
+      end
+    end
+
+    def to_f
+      if @exponent == EXPONENT_MASK
+        if @sign.zero?
+          Float::INFINITY
+        else
+          -Float::INFINITY
+        end
+      else
+        if @exponent.zero?
+          implicit_fraction = 0
+        else
+          implicit_fraction = 1
+        end
+        ((-1) ** @sign) *
+          (2 ** (@exponent - EXPONENT_BIAS)) *
+          (implicit_fraction + @fraction / FRACTION_DENOMINATOR)
+      end
+    end
+
+    def to_uint16
+      (@sign << (EXPONENT_N_BITS + FRACTION_N_BITS)) ^
+        (@exponent << FRACTION_N_BITS) ^
+        @fraction
+    end
+
+    def pack
+      [to_uint16].pack("S")
+    end
+
+    private
+    def deconstruct_float(float)
+      if float > MAX
+        float = Float::INFINITY
+      elsif float < MIN
+        float = -Float::INFINITY
+      end
+      is_infinite = float.infinite?
+      if is_infinite
+        sign = (is_infinite == 1) ? 0 : 1
+        exponent = EXPONENT_MASK
+        fraction = 0
+      elsif float.zero?
+        sign = 0
+        exponent = 0
+        fraction = 0
+      else
+        sign = (float.positive? ? 0 : 1)
+        float_abs = float.abs
+        1.upto(EXPONENT_MASK) do |e|
+          next_exponent_value = 2 ** (e + 1 - EXPONENT_BIAS)
+          next if float_abs > next_exponent_value
+          exponent = e
+          exponent_value = 2 ** (e - EXPONENT_BIAS)
+          fraction =
+            ((float_abs / exponent_value - 1) * FRACTION_DENOMINATOR).round
+          break
+        end
+      end
+      [sign, exponent, fraction]
+    end
+
+    def deconstruct_uint16(uint16)
+      # | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
+      sign = (uint16 >> (EXPONENT_N_BITS + FRACTION_N_BITS))
+      exponent = ((uint16 >> FRACTION_N_BITS) & EXPONENT_MASK)
+      fraction = (uint16 & FRACTION_MASK)
+      [sign, exponent, fraction]
+    end
+  end
+end

data/{test/helper/fixture.rb → lib/arrow/input-referable.rb}

@@ -15,14 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-module Helper
-  module Fixture
-    def fixture_dir
-      Pathname.new(__dir__).join("..", "fixture").expand_path
+module Arrow
+  module InputReferable
+    def refer_input(input)
+      @input = input
     end
 
-    def fixture_path(*components)
-      fixture_dir.join(*components)
+    def share_input(other)
+      return unless defined?(@input)
+      other.refer_input(@input)
     end
   end
 end

data/lib/arrow/jruby/array-builder.rb

@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative "array"
+
+module Arrow
+  module ArrayBuildable
+    ValueVector = org.apache.arrow.vector.ValueVector
+    def buildable?(args)
+      return false if args.size == 1 and args.first.is_a?(ValueVector)
+      super
+    end
+  end
+
+  class ArrayBuilder
+    class << self
+      prepend ArrayBuildable
+    end
+
+    def initialize
+      @vector = self.class::Array::Vector.new("", Arrow.allocator)
+      @vector.allocate_new
+      @index = 0
+    end
+
+    def append_value(value)
+      @vector.set(@index, value)
+      @index += 1
+    end
+
+    def append_values(values, is_valids=nil)
+      if is_valids
+        values.zip(is_valids) do |value, is_valid|
+          if is_valid
+            @vector.set(@index, value)
+          else
+            @vector.set_null(@index)
+          end
+          @index += 1
+        end
+      else
+        values.each do |value|
+          @vector.set(@index, value)
+          @index += 1
+        end
+      end
+    end
+
+    def append_nulls(n)
+      n.times do
+        @vector.set_null(@index)
+        @index += 1
+      end
+    end
+
+    def finish
+      @vector.set_value_count(@index)
+      vector, @vector = @vector, nil
+      self.class::Array.new(vector)
+    end
+  end
+
+  class Int8ArrayBuilder < ArrayBuilder
+    Array = Int8Array
+  end
+
+  class Int32ArrayBuilder < ArrayBuilder
+    Array = Int32Array
+  end
+
+  class FixedSizeBinaryArrayBuilder < ArrayBuilder
+  end
+
+  class Decimal128ArrayBuilder < FixedSizeBinaryArrayBuilder
+  end
+
+  class Decimal256ArrayBuilder < FixedSizeBinaryArrayBuilder
+  end
+
+  class ListArrayBuilder < ArrayBuilder
+  end
+
+  class MapArrayBuilder < ArrayBuilder
+  end
+
+  class StructArrayBuilder < ArrayBuilder
+  end
+
+  class UnionArrayBuilder < ArrayBuilder
+    def append_child(child, filed_name)
+      raise NotImplementedError
+    end
+  end
+
+  class DenseUnionArrayBuilder < UnionArrayBuilder
+  end
+
+  class SparseUnionArrayBuilder < UnionArrayBuilder
+  end
+end

data/lib/arrow/jruby/array.rb

@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative "data-type"
+
+module Arrow
+  class Array
+    VectorAppender = org.apache.arrow.vector.util.VectorAppender
+    VectorEqualsVisitor = org.apache.arrow.vector.compare.VectorEqualsVisitor
+
+    attr_reader :vector
+
+    def initialize(vector)
+      @vector = vector
+    end
+
+    def ==(other_array)
+      return false unless other_array.is_a?(self.class)
+      VectorEqualsVisitor.vector_equals(@vector, other_array.vector)
+    end
+
+    def null?(i)
+      @vector.null?(i)
+    end
+
+    def get_value(i)
+      @vector.get_object(i)
+    end
+
+    def to_s
+      @vector.to_s
+    end
+
+    def inspect
+      super.sub(/>\z/) do
+        " #{to_s}>"
+      end
+    end
+
+    def close
+      @vector.close
+    end
+
+    def length
+      @vector.value_count
+    end
+
+    def value_data_type
+      self.class::ValueDataType.new
+    end
+
+    def values
+      each.to_a
+    end
+
+    def cast(other_value_data_type)
+      other_value_data_type.build_array(to_a)
+    end
+
+    def is_in(values)
+      raise NotImplementedError
+    end
+
+    def concatenate(other_arrays)
+      total_size = length + other_arrays.sum(&:length)
+      vector = self.class::Vector.new("", Arrow.allocator)
+      vector.allocate_new(total_size)
+      appender = VectorAppender.new(vector)
+      @vector.accept(appender, nil)
+      other_arrays.each do |other_array|
+        other_array.vector.accept(appender, nil)
+      end
+      self.class.new(vector)
+    end
+  end
+
+  class Int8Array < Array
+    Vector = org.apache.arrow.vector.SmallIntVector
+    ValueDataType = Int8DataType
+  end
+
+  class Int32Array < Array
+    Vector = org.apache.arrow.vector.IntVector
+    ValueDataType = Int32DataType
+  end
+
+  class FixedSizeBinaryArray < Array
+  end
+
+  class StructArray < Array
+    def fields
+      raise NotImplementedError
+    end
+  end
+end

data/lib/arrow/jruby/chunked-array.rb

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class ChunkedArray
+    def initialize(arrays)
+      @arrays = arrays
+    end
+
+    def n_rows
+      @arrays.sum(&:size)
+    end
+
+    def chunks
+      @arrays
+    end
+
+    def get_chunk(i)
+      @arrays[i]
+    end
+  end
+end

data/lib/arrow/jruby/compression-type.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class CompressionType
+    class << self
+      def values
+        []
+      end
+    end
+  end
+end

data/lib/arrow/jruby/csv-read-options.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class CSVReadOptions
+    def add_column_type(name, type)
+      raise NotImplementedError
+    end
+
+    def delimiter
+      raise NotImplementedError
+    end
+
+    def delimiter=(delimiter)
+      raise NotImplementedError
+    end
+  end
+end

data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb}

@@ -15,22 +15,34 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class MapDataTypeTest < Test::Unit::TestCase
-  sub_test_case(".new") do
-    def setup
-      @key = :int8
-      @item = :string
+module Arrow
+  class DataType
+    def initialize
+      @minor_type = self.class::MinorType
     end
+  end
+
+  class Int8DataType < DataType
+    MinorType = org.apache.arrow.vector.types.Types::MinorType::SMALLINT
+  end
 
-    test("ordered arguments") do
-      assert_equal("map<int8, string>",
-                   Arrow::MapDataType.new(@key, @item).to_s)
+  class Int32DataType < DataType
+    MinorType = org.apache.arrow.vector.types.Types::MinorType::INT
+  end
+
+  class Decimal128DataType < DataType
+    class << self
+      def max_precision
+        38
+      end
     end
+  end
 
-    test("description") do
-      assert_equal("map<int8, string>",
-                   Arrow::MapDataType.new(key: @key,
-                                          item: @item).to_s)
+  class Decimal256DataType < DataType
+    class << self
+      def max_precision
+        76
+      end
     end
   end
 end