red-arrow 10.0.1 → 11.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c8cc21ba05de4956b7dd412963d2d39eb5f5d31566c6891ae4388064553baa97
-  data.tar.gz: a7c1bfa1d73f3a4ab8403e347902ffd6f754f6964d3054788a79d1022b32520b
+  metadata.gz: 23f72b7016d780c208dd8f8cbc627becce73e25754f83a62a4a4d3f4bb60a5e3
+  data.tar.gz: 23f57e383e26d322e9fa81efed1e8e5545bd5b1160075f4a0b8553efda98af93
 SHA512:
-  metadata.gz: e33a2acb65472b70c99348fec8b472ebeac48bbd3ba3c6aabd91481ed214b07e41fa3c6c3090ecd9ed545ee54cbe2fb09a0a7d730421a816afe6c167b00b6a5a
-  data.tar.gz: 47ab431103a9bcb0f5b4af955013b40415a582723b55640cd6b82a4b8af5ef775ed974e9f62d235cc81c296c0d7a0438edf805f3ed1ab442cae58501cb73625a
+  metadata.gz: 2d157d2d56dbca00a2f4a0eadb831a19a7a7124c3c3c8c7675536e4597f2a1447f047eec48654256eff3148457d09f1d3e375a095cf152051ef6603cd8bf25bf
+  data.tar.gz: 958f320a92981a6ed7f84fcab92497ac1038a038de302419e9f57f83e3ca4d02d682ec05a129ac63c9ed6e613725619839b46cc589938a56573e88182a8002d4

data/ext/arrow/converters.hpp

@@ -106,10 +106,34 @@ namespace red_arrow {
       return ULL2NUM(array.Value(i));
     }
 
-    // TODO
-    // inline VALUE convert(const arrow::HalfFloatArray& array,
-    //                      const int64_t i) {
-    // }
+    inline VALUE convert(const arrow::HalfFloatArray& array,
+                         const int64_t i) {
+      const auto value = array.Value(i);
+      // | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
+      constexpr auto exponent_n_bits = 5;
+      static const auto exponent_mask =
+        static_cast<uint32_t>(std::pow(2.0, exponent_n_bits) - 1);
+      constexpr auto exponent_bias = 15;
+      constexpr auto fraction_n_bits = 10;
+      static const auto fraction_mask =
+        static_cast<uint32_t>(std::pow(2.0, fraction_n_bits)) - 1;
+      static const auto fraction_denominator = std::pow(2.0, fraction_n_bits);
+      const auto sign = value >> (exponent_n_bits + fraction_n_bits);
+      const auto exponent = (value >> fraction_n_bits) & exponent_mask;
+      const auto fraction = value & fraction_mask;
+      if (exponent == exponent_mask) {
+        if (sign == 0) {
+          return DBL2NUM(HUGE_VAL);
+        } else {
+          return DBL2NUM(-HUGE_VAL);
+        }
+      } else {
+        const auto implicit_fraction = (exponent == 0) ? 0 : 1;
+        return DBL2NUM(((sign == 0) ? 1 : -1) *
+                       std::pow(2.0, exponent - exponent_bias) *
+                       (implicit_fraction + fraction / fraction_denominator));
+      }
+    }
 
     inline VALUE convert(const arrow::FloatArray& array,
                          const int64_t i) {
@@ -320,8 +344,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -427,8 +450,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -530,8 +552,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -634,8 +655,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -761,8 +781,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)

data/ext/arrow/extconf.rb

@@ -38,8 +38,6 @@ checking_for(checking_message("Homebrew")) do
   end
 end
 
-$CXXFLAGS += " -std=c++17 "
-
 unless required_pkg_config_package([
                                      "arrow",
                                      Arrow::Version::MAJOR,
@@ -77,4 +75,18 @@ end
   add_depend_package_path(name, source_dir, build_dir)
 end
 
+case RUBY_PLATFORM
+when /darwin/
+  symbols_in_external_bundles = [
+    "_rbgerr_gerror2exception",
+    "_rbgobj_instance_from_ruby_object",
+  ]
+  symbols_in_external_bundles.each do |symbol|
+    $DLDFLAGS << " -Wl,-U,#{symbol}"
+  end
+  mmacosx_version_min = "-mmacosx-version-min=10.14"
+  $CFLAGS << " #{mmacosx_version_min}"
+  $CXXFLAGS << " #{mmacosx_version_min}"
+end
+
 create_makefile("arrow")

data/ext/arrow/raw-records.cpp

@@ -84,8 +84,7 @@ namespace red_arrow {
       VISIT(UInt16)
       VISIT(UInt32)
       VISIT(UInt64)
-      // TODO
-      // VISIT(HalfFloat)
+      VISIT(HalfFloat)
       VISIT(Float)
       VISIT(Double)
       VISIT(Binary)

data/ext/arrow/values.cpp

@@ -65,8 +65,7 @@ namespace red_arrow {
       VISIT(UInt16)
       VISIT(UInt32)
       VISIT(UInt64)
-      // TODO
-      // VISIT(HalfFloat)
+      VISIT(HalfFloat)
       VISIT(Float)
       VISIT(Double)
       VISIT(Binary)

data/lib/arrow/array.rb

@@ -22,6 +22,7 @@ module Arrow
     include ArrayComputable
     include GenericFilterable
     include GenericTakeable
+    include InputReferable
 
     class << self
       def new(*args)
@@ -115,6 +116,10 @@ module Arrow
       self
     end
 
+    def to_arrow_chunked_array
+      ChunkedArray.new([self])
+    end
+
     alias_method :value_data_type_raw, :value_data_type
     def value_data_type
       @value_data_type ||= value_data_type_raw

data/lib/arrow/chunked-array.rb

@@ -22,6 +22,19 @@ module Arrow
     include ArrayComputable
     include GenericFilterable
     include GenericTakeable
+    include InputReferable
+
+    def to_arrow
+      self
+    end
+
+    def to_arrow_array
+      combine
+    end
+
+    def to_arrow_chunked_array
+      self
+    end
 
     alias_method :size, :n_rows
     unless method_defined?(:length)
@@ -30,7 +43,16 @@ module Arrow
 
     alias_method :chunks_raw, :chunks
     def chunks
-      @chunks ||= chunks_raw
+      @chunks ||= chunks_raw.tap do |_chunks|
+        _chunks.each do |chunk|
+          share_input(chunk)
+        end
+      end
+    end
+
+    alias_method :get_chunk_raw, :get_chunk
+    def get_chunk(i)
+      chunks[i]
     end
 
     def null?(i)

data/lib/arrow/column-containable.rb

@@ -143,5 +143,14 @@ module Arrow
         find_column(selector)
       end
     end
+
+    # Return column names in this object.
+    #
+    # @return [::Array<String>] column names.
+    #
+    # @since 11.0.0
+    def column_names
+      @column_names ||= columns.collect(&:name)
+    end
   end
 end

data/lib/arrow/column.rb

@@ -27,6 +27,7 @@ module Arrow
       @index = index
       @field = @container.schema[@index]
       @data = @container.get_column_data(@index)
+      @container.share_input(@data)
     end
 
     def name

data/lib/arrow/half-float-array-builder.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloatArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      case value
+      when Float
+        HalfFloat.new(value).to_uint16
+      when HalfFloat
+        value.to_uint16
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/half-float-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloatArray
+    def get_value(i)
+      HalfFloat.new(get_raw_value(i)).to_f
+    end
+  end
+end

data/lib/arrow/half-float.rb

@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloat
+    MAX = 65504
+    MIN = -65504
+    EXPONENT_N_BITS = 5
+    EXPONENT_MASK = (2 ** EXPONENT_N_BITS) - 1
+    EXPONENT_BIAS = 15
+    FRACTION_N_BITS = 10
+    FRACTION_MASK = (2 ** FRACTION_N_BITS) - 1
+    FRACTION_DENOMINATOR = 2.0 ** FRACTION_N_BITS
+
+    attr_reader :sign
+    attr_reader :exponent
+    attr_reader :fraction
+    def initialize(*args)
+      n_args = args.size
+      case n_args
+      when 1
+        if args[0].is_a?(Float)
+          @sign, @exponent, @fraction = deconstruct_float(args[0])
+        else
+          @sign, @exponent, @fraction = deconstruct_uint16(args[0])
+        end
+      when 3
+        @sign, @exponent, @fraction = *args
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 1 or 3)"
+        raise ArgumentError, message
+      end
+    end
+
+    def to_f
+      if @exponent == EXPONENT_MASK
+        if @sign.zero?
+          Float::INFINITY
+        else
+          -Float::INFINITY
+        end
+      else
+        if @exponent.zero?
+          implicit_fraction = 0
+        else
+          implicit_fraction = 1
+        end
+        ((-1) ** @sign) *
+          (2 ** (@exponent - EXPONENT_BIAS)) *
+          (implicit_fraction + @fraction / FRACTION_DENOMINATOR)
+      end
+    end
+
+    def to_uint16
+      (@sign << (EXPONENT_N_BITS + FRACTION_N_BITS)) ^
+        (@exponent << FRACTION_N_BITS) ^
+        @fraction
+    end
+
+    def pack
+      [to_uint16].pack("S")
+    end
+
+    private
+    def deconstruct_float(float)
+      if float > MAX
+        float = Float::INFINITY
+      elsif float < MIN
+        float = -Float::INFINITY
+      end
+      is_infinite = float.infinite?
+      if is_infinite
+        sign = (is_infinite == 1) ? 0 : 1
+        exponent = EXPONENT_MASK
+        fraction = 0
+      elsif float.zero?
+        sign = 0
+        exponent = 0
+        fraction = 0
+      else
+        sign = (float.positive? ? 0 : 1)
+        float_abs = float.abs
+        1.upto(EXPONENT_MASK) do |e|
+          next_exponent_value = 2 ** (e + 1 - EXPONENT_BIAS)
+          next if float_abs > next_exponent_value
+          exponent = e
+          exponent_value = 2 ** (e - EXPONENT_BIAS)
+          fraction =
+            ((float_abs / exponent_value - 1) * FRACTION_DENOMINATOR).round
+          break
+        end
+      end
+      [sign, exponent, fraction]
+    end
+
+    def deconstruct_uint16(uint16)
+      # | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
+      sign = (uint16 >> (EXPONENT_N_BITS + FRACTION_N_BITS))
+      exponent = ((uint16 >> FRACTION_N_BITS) & EXPONENT_MASK)
+      fraction = (uint16 & FRACTION_MASK)
+      [sign, exponent, fraction]
+    end
+  end
+end

data/lib/arrow/input-referable.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module InputReferable
+    def refer_input(input)
+      @input = input
+    end
+
+    def share_input(other)
+      return unless defined?(@input)
+      other.refer_input(@input)
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -39,6 +39,7 @@ module Arrow
       require "arrow/field-containable"
       require "arrow/generic-filterable"
       require "arrow/generic-takeable"
+      require "arrow/input-referable"
       require "arrow/record-containable"
       require "arrow/symbol-values-appendable"
 
@@ -81,6 +82,9 @@ module Arrow
       require "arrow/fixed-size-binary-array-builder"
       require "arrow/function"
       require "arrow/group"
+      require "arrow/half-float"
+      require "arrow/half-float-array"
+      require "arrow/half-float-array-builder"
       require "arrow/list-array-builder"
       require "arrow/list-data-type"
       require "arrow/map-array"
@@ -196,6 +200,7 @@ module Arrow
            "Arrow::Date64Array",
            "Arrow::Decimal128Array",
            "Arrow::Decimal256Array",
+           "Arrow::HalfFloatArray",
            "Arrow::Time32Array",
            "Arrow::Time64Array",
            "Arrow::TimestampArray"

data/lib/arrow/raw-table-converter.rb

@@ -35,14 +35,16 @@ module Arrow
         fields = []
         @values = []
         @raw_table.each do |name, array|
-          if array.respond_to?(:to_arrow_array)
-            array = array.to_arrow_array
+          if array.respond_to?(:to_arrow_chunked_array)
+            chunked_array = array.to_arrow_chunked_array
+          elsif array.respond_to?(:to_arrow_array)
+            chunked_array = ChunkedArray.new([array.to_arrow_array])
           else
             array = array.to_ary if array.respond_to?(:to_ary)
-            array = ArrayBuilder.build(array)
+            chunked_array = ChunkedArray.new([ArrayBuilder.build(array)])
           end
-          fields << Field.new(name.to_s, array.value_data_type)
-          @values << array
+          fields << Field.new(name.to_s, chunked_array.value_data_type)
+          @values << chunked_array
         end
         @schema = Schema.new(fields)
       end

data/lib/arrow/record-batch.rb

@@ -19,9 +19,11 @@ require "arrow/raw-table-converter"
 
 module Arrow
   class RecordBatch
+    include Enumerable
+
     include ColumnContainable
+    include InputReferable
     include RecordContainable
-    include Enumerable
 
     class << self
       def new(*args)
@@ -56,7 +58,9 @@ module Arrow
     #
     # @since 0.12.0
     def to_table
-      Table.new(schema, [self])
+      table = Table.new(schema, [self])
+      share_input(table)
+      table
     end
 
     def respond_to_missing?(name, include_private)

data/lib/arrow/table-loader.rb

@@ -161,7 +161,7 @@ module Arrow
         record_batches << record_batch
       end
       table = Table.new(schema, record_batches)
-      table.instance_variable_set(:@input, input)
+      table.refer_input(input)
       table
     end
 
@@ -211,7 +211,7 @@ module Arrow
           field_indexes = @options[:field_indexes]
           reader.set_field_indexes(field_indexes) if field_indexes
           table = reader.read_stripes
-          table.instance_variable_set(:@input, input)
+          table.refer_input(input)
           table
         end
       end
@@ -245,7 +245,7 @@ module Arrow
       open_input_stream do |input|
         reader = FeatherFileReader.new(input)
         table = reader.read
-        table.instance_variable_set(:@input, input)
+        table.refer_input(input)
         table
       end
     end
@@ -254,7 +254,7 @@ module Arrow
       open_input_stream do |input|
         reader = JSONReader.new(input)
         table = reader.read
-        table.instance_variable_set(:@input, input)
+        table.refer_input(input)
         table
       end
     end

data/lib/arrow/table-saver.rb

@@ -51,6 +51,7 @@ module Arrow
         raise ArgumentError, message
       end
       __send__(custom_save_method)
+      @table
     end
 
     private

data/lib/arrow/table.rb

@@ -22,6 +22,7 @@ module Arrow
     include ColumnContainable
     include GenericFilterable
     include GenericTakeable
+    include InputReferable
     include RecordContainable
 
     class << self
@@ -188,6 +189,7 @@ module Arrow
 
       reader = TableBatchReader.new(self)
       while record_batch = reader.read_next
+        share_input(record_batch)
         yield(record_batch)
       end
     end
@@ -346,10 +348,12 @@ module Arrow
         end
       end
       if sliced_tables.size > 1
-        sliced_tables[0].concatenate(sliced_tables[1..-1])
+        sliced_table = sliced_tables[0].concatenate(sliced_tables[1..-1])
       else
-        sliced_tables[0]
+        sliced_table = sliced_tables[0]
       end
+      share_input(sliced_table)
+      sliced_table
     end
 
     # TODO
@@ -401,7 +405,9 @@ module Arrow
         new_fields << new_column[:field]
         new_arrays << new_column[:data]
       end
-      self.class.new(new_fields, new_arrays)
+      table = self.class.new(new_fields, new_arrays)
+      share_input(table)
+      table
     end
 
     alias_method :remove_column_raw, :remove_column
@@ -423,7 +429,9 @@ module Arrow
           raise IndexError.new(message)
         end
       end
-      remove_column_raw(index)
+      table = remove_column_raw(index)
+      share_input(table)
+      table
     end
 
     # Experimental
@@ -445,43 +453,59 @@ module Arrow
       packed_arrays = columns.collect do |column|
         column.data.pack
       end
-      self.class.new(schema, packed_arrays)
+      table = self.class.new(schema, packed_arrays)
+      share_input(table)
+      table
     end
 
-    # @overload join(right, key, type: :inner, left_outputs: nil, right_outputs: nil)
-    #   @!macro join_common_before
-    #     @param right [Arrow::Table] The right table.
+    # Join another Table by matching with keys.
+    #
+    # @!macro join_common_before
+    #   @param right [Arrow::Table] The right table.
+    #
+    #   Join columns with `right` on join key columns.
+    #
+    # @!macro join_common_after
+    #   @param type [Arrow::JoinType] How to join.
+    #   @param left_outputs [::Array<String, Symbol>] Output columns in
+    #     `self`.
+    #
+    #     If both of `left_outputs` and `right_outputs` aren't
+    #     specified, all columns in `self` and `right` are
+    #     outputted.
+    #   @param right_outputs [::Array<String, Symbol>] Output columns in
+    #     `right`.
     #
-    #     Join columns with `right` on join key columns.
+    #     If both of `left_outputs` and `right_outputs` aren't
+    #     specified, all columns in `self` and `right` are
+    #     outputted.
+    #   @return [Arrow::Table]
+    #     The joined `Arrow::Table`.
     #
-    #   @!macro join_common_after
-    #     @param type [Arrow::JoinType] How to join.
-    #     @param left_outputs [::Array<String, Symbol>] Output columns in
-    #       `self`.
+    # @overload join(right, type: :inner, left_outputs: nil, right_outputs: nil)
+    #   If key(s) are not supplied, common keys in self and right are used.
     #
-    #       If both of `left_outputs` and `right_outputs` aren't
-    #       specified, all columns in `self` and `right` are
-    #       outputted.
-    #     @param right_outputs [::Array<String, Symbol>] Output columns in
-    #       `right`.
+    #   @macro join_common_before
+    #   @macro join_common_after
     #
-    #       If both of `left_outputs` and `right_outputs` aren't
-    #       specified, all columns in `self` and `right` are
-    #       outputted.
-    #     @return [Arrow::Table]
-    #       The joined `Arrow::Table`.
+    # @since 11.0.0
+    #
+    # @overload join(right, key, type: :inner, left_outputs: nil, right_outputs: nil)
+    #   Join right by a key.
     #
     #   @macro join_common_before
     #   @param key [String, Symbol] A join key.
     #   @macro join_common_after
     #
     # @overload join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+    #   Join right by keys.
     #
     #   @macro join_common_before
     #   @param keys [::Array<String, Symbol>] Join keys.
     #   @macro join_common_after
     #
     # @overload join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+    #   Join right by a key or keys mapped by a hash.
     #
     #   @macro join_common_before
     #   @param keys [Hash] Specify join keys in `self` and `right` separately.
@@ -492,7 +516,8 @@ module Arrow
     #   @macro join_common_after
     #
     # @since 7.0.0
-    def join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
+    def join(right, keys=nil, type: :inner, left_outputs: nil, right_outputs: nil)
+      keys ||= (column_names & right.column_names)
       plan = ExecutePlan.new
       left_node = plan.build_source_node(self)
       right_node = plan.build_source_node(right)
@@ -523,7 +548,9 @@ module Arrow
       plan.start
       plan.wait
       reader = sink_node_options.get_reader(hash_join_node.output_schema)
-      reader.read_all
+      table = reader.read_all
+      share_input(table)
+      table
     end
 
     alias_method :to_s_raw, :to_s

data/lib/arrow/tensor.rb

@@ -160,5 +160,9 @@ module Arrow
                                       nil,
                                       0)
     end
+
+    def to_arrow_chunked_array
+      ChunkedArray.new([to_arrow_array])
+    end
   end
 end

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "10.0.1"
+  VERSION = "11.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -47,7 +47,7 @@ Gem::Specification.new do |spec|
   spec.extensions = ["ext/arrow/extconf.rb"]
 
   spec.add_runtime_dependency("bigdecimal", ">= 3.1.0")
-  spec.add_runtime_dependency("extpp", ">= 0.0.7")
+  spec.add_runtime_dependency("extpp", ">= 0.1.1")
   spec.add_runtime_dependency("gio2", ">= 3.5.0")
   spec.add_runtime_dependency("native-package-installer")
   spec.add_runtime_dependency("pkg-config")

data/test/raw-records/test-basic-arrays.rb

@@ -117,6 +117,16 @@ module RawRecordsBasicArraysTests
     assert_equal(records, target.raw_records)
   end
 
+  def test_half_float
+    records = [
+      [-1.5],
+      [nil],
+      [1.5],
+    ]
+    target = build({column: :half_float}, records)
+    assert_equal(records, target.raw_records)
+  end
+
   def test_float
     records = [
       [-1.0],

data/test/test-data-type.rb

@@ -54,7 +54,8 @@ class DataTypeTest < Test::Unit::TestCase
         "abstract type: <:floating_point>: " +
         "use one of not abstract type: [" +
         "Arrow::DoubleDataType, " +
-        "Arrow::FloatDataType]"
+        "Arrow::FloatDataType, " +
+        "Arrow::HalfFloatDataType]"
       assert_raise(ArgumentError.new(message)) do
         Arrow::DataType.resolve(:floating_point)
       end

data/test/test-half-float-array.rb

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class HalfFloatArrayTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("Float") do
+      array = Arrow::HalfFloatArray.new([1.5])
+      assert_equal([1.5], array.to_a)
+    end
+
+    test("Integer") do
+      one_half = Arrow::HalfFloat.new(1.5)
+      array = Arrow::HalfFloatArray.new([one_half.to_uint16])
+      assert_equal([one_half.to_f], array.to_a)
+    end
+
+    test("HalfFloat") do
+      one_half = Arrow::HalfFloat.new(1.5)
+      array = Arrow::HalfFloatArray.new([one_half])
+      assert_equal([one_half.to_f], array.to_a)
+    end
+  end
+
+  test("#[]") do
+    one_half = Arrow::HalfFloat.new(1.5)
+    array = Arrow::HalfFloatArray.new([one_half.to_uint16])
+    assert_equal(one_half.to_f, array[0])
+  end
+end

data/test/test-half-float.rb

@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class HalfFloatTest < Test::Unit::TestCase
+  sub_test_case(".new") do
+    test("Array") do
+      positive_infinity = Arrow::HalfFloat.new(0b1, 0b11111, 0b0000000000)
+      assert_equal([0b1, 0b11111, 0b0000000000],
+                   [
+                     positive_infinity.sign,
+                     positive_infinity.exponent,
+                     positive_infinity.fraction,
+                   ])
+    end
+
+    test("Integer - 0") do
+      zero = Arrow::HalfFloat.new(0)
+      assert_equal([0b0, 0b00000, 0b0000000000],
+                   [
+                     zero.sign,
+                     zero.exponent,
+                     zero.fraction,
+                   ])
+    end
+
+    test("Integer - +infinity") do
+      positive_infinity = Arrow::HalfFloat.new(0x7c00)
+      assert_equal([0b0, 0b11111, 0b0000000000],
+                   [
+                     positive_infinity.sign,
+                     positive_infinity.exponent,
+                     positive_infinity.fraction,
+                   ])
+    end
+
+    test("Integer - -infinity") do
+      negative_infinity = Arrow::HalfFloat.new(0xfc00)
+      assert_equal([0b1, 0b11111, 0b0000000000],
+                   [
+                     negative_infinity.sign,
+                     negative_infinity.exponent,
+                     negative_infinity.fraction,
+                   ])
+    end
+
+    test("Integer - 1/3") do
+      one_thirds = Arrow::HalfFloat.new(0x3555)
+      assert_equal([0b0, 0b01101, 0b0101010101],
+                   [
+                     one_thirds.sign,
+                     one_thirds.exponent,
+                     one_thirds.fraction,
+                   ])
+    end
+
+    test("Float - 0") do
+      zero = Arrow::HalfFloat.new(0.0)
+      assert_equal([0b0, 0b00000, 0b0000000000],
+                   [
+                     zero.sign,
+                     zero.exponent,
+                     zero.fraction,
+                   ])
+    end
+
+    test("Float - too large") do
+      positive_infinity = Arrow::HalfFloat.new(65504.1)
+      assert_equal([0b0, 0b11111, 0b0000000000],
+                   [
+                     positive_infinity.sign,
+                     positive_infinity.exponent,
+                     positive_infinity.fraction,
+                   ])
+    end
+
+    test("Float - +infinity") do
+      positive_infinity = Arrow::HalfFloat.new(Float::INFINITY)
+      assert_equal([0b0, 0b11111, 0b0000000000],
+                   [
+                     positive_infinity.sign,
+                     positive_infinity.exponent,
+                     positive_infinity.fraction,
+                   ])
+    end
+
+    test("Float - too small") do
+      negative_infinity = Arrow::HalfFloat.new(-65504.1)
+      assert_equal([0b1, 0b11111, 0b0000000000],
+                   [
+                     negative_infinity.sign,
+                     negative_infinity.exponent,
+                     negative_infinity.fraction,
+                   ])
+    end
+
+    test("Float - -infinity") do
+      negative_infinity = Arrow::HalfFloat.new(-Float::INFINITY)
+      assert_equal([0b1, 0b11111, 0b0000000000],
+                   [
+                     negative_infinity.sign,
+                     negative_infinity.exponent,
+                     negative_infinity.fraction,
+                   ])
+    end
+
+    test("Float - 1/3") do
+      one_thirds = Arrow::HalfFloat.new((2 ** -2) * (1 + 341 / 1024.0))
+      assert_equal([0b0, 0b01101, 0b0101010101],
+                   [
+                     one_thirds.sign,
+                     one_thirds.exponent,
+                     one_thirds.fraction,
+                   ])
+    end
+  end
+end

data/test/test-table.rb

@@ -41,8 +41,25 @@ class TableTest < Test::Unit::TestCase
   end
 
   sub_test_case(".new") do
+    test("{Symbol: Arrow::Array}") do
+      schema = Arrow::Schema.new(numbers: :int64)
+      assert_equal(Arrow::Table.new(schema,
+                                    [Arrow::Int64Array.new([1, 2, 3])]),
+                   Arrow::Table.new(numbers: Arrow::Int64Array.new([1, 2, 3])))
+    end
+
+    test("{Symbol: Arrow::ChunkedArray}") do
+      chunked_array = Arrow::ChunkedArray.new([Arrow::Int64Array.new([1, 2, 3])])
+      schema = Arrow::Schema.new(numbers: :int64)
+      assert_equal(Arrow::Table.new(schema,
+                                    [Arrow::Int64Array.new([1, 2, 3])]),
+                   Arrow::Table.new(numbers: chunked_array))
+    end
+
     test("{Symbol: Arrow::Tensor}") do
-      assert_equal(Arrow::Table.new(numbers: Arrow::UInt8Array.new([1, 2, 3])),
+      schema = Arrow::Schema.new(numbers: :uint8)
+      assert_equal(Arrow::Table.new(schema,
+                                    [Arrow::UInt8Array.new([1, 2, 3])]),
                    Arrow::Table.new(numbers: Arrow::Tensor.new([1, 2, 3])))
     end
 
@@ -51,7 +68,8 @@ class TableTest < Test::Unit::TestCase
       def array_like.to_ary
         [1, 2, 3]
       end
-      assert_equal(Arrow::Table.new(numbers: Arrow::UInt8Array.new([1, 2, 3])),
+      schema = Arrow::Schema.new(numbers: :uint8)
+      assert_equal(Arrow::Table.new(schema, [Arrow::UInt8Array.new([1, 2, 3])]),
                    Arrow::Table.new(numbers: array_like))
     end
   end
@@ -584,6 +602,18 @@ class TableTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case("#column_names") do
+    test("unique") do
+      table = Arrow::Table.new(a: [1], b: [2], c: [3])
+      assert_equal(%w[a b c], table.column_names)
+    end
+
+    test("duplicated") do
+      table = Arrow::Table.new([["a", [1, 2, 3]], ["a", [4, 5, 6]]])
+      assert_equal(%w[a a], table.column_names)
+    end
+  end
+
   sub_test_case("#save and .load") do
     module SaveLoadFormatTests
       def test_default
@@ -690,6 +720,11 @@ class TableTest < Test::Unit::TestCase
                                            schema: @table.schema))
           end
 
+          test("csv, return value") do
+            output = create_output(".csv")
+            assert_equal(@table, @table.save(output))
+          end
+
           test("csv.gz") do
             output = create_output(".csv.gz")
             @table.save(output)
@@ -830,6 +865,76 @@ chris\t-1
         end
       end
     end
+
+    sub_test_case("GC") do
+      def setup
+        table = Arrow::Table.new(integer: [1, 2, 3],
+                                 string: ["a", "b", "c"])
+        @buffer = Arrow::ResizableBuffer.new(1024)
+        table.save(@buffer, format: :arrow)
+        @loaded_table = Arrow::Table.load(@buffer)
+      end
+
+      def test_chunked_array
+        chunked_array = @loaded_table[0].data
+        assert_equal(@buffer,
+                     chunked_array.instance_variable_get(:@input).buffer)
+      end
+
+      def test_array
+        array = @loaded_table[0].data.chunks[0]
+        assert_equal(@buffer,
+                     array.instance_variable_get(:@input).buffer)
+      end
+
+      def test_record_batch
+        record_batch = @loaded_table.each_record_batch.first
+        assert_equal(@buffer,
+                     record_batch.instance_variable_get(:@input).buffer)
+      end
+
+      def test_record_batch_array
+        array = @loaded_table.each_record_batch.first[0].data
+        assert_equal(@buffer,
+                     array.instance_variable_get(:@input).buffer)
+      end
+
+      def test_record_batch_table
+        table = @loaded_table.each_record_batch.first.to_table
+        assert_equal(@buffer,
+                     table.instance_variable_get(:@input).buffer)
+      end
+
+      def test_slice
+        table = @loaded_table.slice(0..-1)
+        assert_equal(@buffer,
+                     table.instance_variable_get(:@input).buffer)
+      end
+
+      def test_merge
+        table = @loaded_table.merge({})
+        assert_equal(@buffer,
+                     table.instance_variable_get(:@input).buffer)
+      end
+
+      def test_remove_column
+        table = @loaded_table.remove_column(0)
+        assert_equal(@buffer,
+                     table.instance_variable_get(:@input).buffer)
+      end
+
+      def test_pack
+        table = @loaded_table.pack
+        assert_equal(@buffer,
+                     table.instance_variable_get(:@input).buffer)
+      end
+
+      def test_join
+        table = @loaded_table.join(@loaded_table, :integer)
+        assert_equal(@buffer,
+                     table.instance_variable_get(:@input).buffer)
+      end
+    end
   end
 
   test("#pack") do
@@ -1026,6 +1131,20 @@ visible: false
   end
 
   sub_test_case("#join") do
+    test("no keys") do
+      table1 = Arrow::Table.new(key: [1, 2, 3],
+                                number: [10, 20, 30])
+      table2 = Arrow::Table.new(key: [3, 1],
+                                string: ["three", "one"])
+      assert_equal(Arrow::Table.new([
+                                      ["key", [1, 3]],
+                                      ["number", [10, 30]],
+                                      ["key", [1, 3]],
+                                      ["string", ["one", "three"]],
+                                    ]),
+                   table1.join(table2))
+    end
+
     test("keys: String") do
       table1 = Arrow::Table.new(key: [1, 2, 3],
                                 number: [10, 20, 30])
@@ -1083,7 +1202,9 @@ visible: false
                                       ["right_key", [1, 3]],
                                       ["string", ["one", "three"]],
                                     ]),
-                   table1.join(table2, {left: "left_key", right: :right_key}))
+                   table1.join(table2,
+                               {left: "left_key", right: :right_key},
+                               type: :inner))
     end
 
     test("keys: {left: [String, Symbol], right: [Symbol, String]}") do
@@ -1105,7 +1226,8 @@ visible: false
                                {
                                  left: ["left_key1", :left_key2],
                                  right: [:right_key1, "right_key2"],
-                               }))
+                               },
+                               type: :inner))
     end
 
     test("type:") do

data/test/values/test-basic-arrays.rb

@@ -107,6 +107,16 @@ module ValuesBasicArraysTests
     assert_equal(values, target.values)
   end
 
+  def test_half_float
+    values = [
+      -1.5,
+      nil,
+      1.5,
+    ]
+    target = build(Arrow::HalfFloatArray.new(values))
+    assert_equal(values, target.values)
+  end
+
   def test_float
     values = [
       -1.0,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 10.0.1
+  version: 11.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-10 00:00:00.000000000 Z
+date: 2023-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.7
+        version: 0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.7
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: gio2
   requirement: !ruby/object:Gem::Requirement
@@ -152,6 +152,10 @@ files:
 - lib/arrow/generic-filterable.rb
 - lib/arrow/generic-takeable.rb
 - lib/arrow/group.rb
+- lib/arrow/half-float-array-builder.rb
+- lib/arrow/half-float-array.rb
+- lib/arrow/half-float.rb
+- lib/arrow/input-referable.rb
 - lib/arrow/list-array-builder.rb
 - lib/arrow/list-data-type.rb
 - lib/arrow/loader.rb
@@ -263,6 +267,8 @@ files:
 - test/test-float-scalar.rb
 - test/test-function.rb
 - test/test-group.rb
+- test/test-half-float-array.rb
+- test/test-half-float.rb
 - test/test-list-array-builder.rb
 - test/test-list-array.rb
 - test/test-list-data-type.rb
@@ -308,7 +314,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata:
-  msys2_mingw_dependencies: arrow>=10.0.1
+  msys2_mingw_dependencies: arrow>=11.0.0
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -324,7 +330,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.15
+rubygems_version: 3.5.0.dev
 signing_key:
 specification_version: 4
 summary: Red Arrow is the Ruby bindings of Apache Arrow
@@ -383,6 +389,8 @@ test_files:
 - test/test-float-scalar.rb
 - test/test-function.rb
 - test/test-group.rb
+- test/test-half-float-array.rb
+- test/test-half-float.rb
 - test/test-list-array-builder.rb
 - test/test-list-array.rb
 - test/test-list-data-type.rb