red-arrow 10.0.1 → 12.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c8cc21ba05de4956b7dd412963d2d39eb5f5d31566c6891ae4388064553baa97
-  data.tar.gz: a7c1bfa1d73f3a4ab8403e347902ffd6f754f6964d3054788a79d1022b32520b
+  metadata.gz: 583b384182a9906a4fcd723cb67b0600b07dd962a1d45e114619014a396800c6
+  data.tar.gz: 97b7a63c52c678b7255a520d768f59fde993ffd2fd9014f24b0b946940f656b8
 SHA512:
-  metadata.gz: e33a2acb65472b70c99348fec8b472ebeac48bbd3ba3c6aabd91481ed214b07e41fa3c6c3090ecd9ed545ee54cbe2fb09a0a7d730421a816afe6c167b00b6a5a
-  data.tar.gz: 47ab431103a9bcb0f5b4af955013b40415a582723b55640cd6b82a4b8af5ef775ed974e9f62d235cc81c296c0d7a0438edf805f3ed1ab442cae58501cb73625a
+  metadata.gz: ab37d28349e4a3b23b629c4a2431a48e6b0a0b4884fb0274a484965e5c2ed551cc5b7b998433940c8e992135803f65a4c3b0555abebd6478fdaeb665f70e3f25
+  data.tar.gz: cfb8de7ce8d9b9cabb76f016e51bde489f9c58fc42253612395082a9ccab79001f56978d102ef18ac93f31e696406980304f2d4e40af46e8b2ba041363308920

data/README.md

@@ -25,9 +25,9 @@ Red Arrow is the Ruby bindings of Apache Arrow. Red Arrow is based on GObject In
 
 [GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection) is a middleware for language bindings of C library. GObject Introspection can generate language bindings automatically at runtime.
 
-Red Arrow uses [Apache Arrow GLib](https://github.com/apache/arrow/tree/master/c_glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Arrow.
+Red Arrow uses [Apache Arrow GLib](https://github.com/apache/arrow/tree/main/c_glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Arrow.
 
-Apache Arrow GLib is a C wrapper for [Apache Arrow C++](https://github.com/apache/arrow/tree/master/cpp). GObject Introspection can't use Apache Arrow C++ directly. Apache Arrow GLib is a bridge between Apache Arrow C++ and GObject Introspection.
+Apache Arrow GLib is a C wrapper for [Apache Arrow C++](https://github.com/apache/arrow/tree/main/cpp). GObject Introspection can't use Apache Arrow C++ directly. Apache Arrow GLib is a bridge between Apache Arrow C++ and GObject Introspection.
 
 gobject-introspection gem is a Ruby bindings of GObject Introspection. Red Arrow uses GObject Introspection via gobject-introspection gem.
 
@@ -56,7 +56,7 @@ table.save("/dev/shm/data-processed.arrow")
 Note that you need to install Apache Arrow C++/GLib at master before preparing Red Arrow. See also:
 
   * For Apache Arrow C++: https://arrow.apache.org/docs/developers/cpp/building.html
-  * For Apache Arrow GLib: https://github.com/apache/arrow/blob/master/c_glib/README.md
+  * For Apache Arrow GLib: https://github.com/apache/arrow/blob/main/c_glib/README.md
 
 ```console
 $ cd ruby/red-arrow

data/ext/arrow/converters.hpp

@@ -106,10 +106,34 @@ namespace red_arrow {
       return ULL2NUM(array.Value(i));
     }
 
-    // TODO
-    // inline VALUE convert(const arrow::HalfFloatArray& array,
-    //                      const int64_t i) {
-    // }
+    inline VALUE convert(const arrow::HalfFloatArray& array,
+                         const int64_t i) {
+      const auto value = array.Value(i);
+      // | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
+      constexpr auto exponent_n_bits = 5;
+      static const auto exponent_mask =
+        static_cast<uint32_t>(std::pow(2.0, exponent_n_bits) - 1);
+      constexpr auto exponent_bias = 15;
+      constexpr auto fraction_n_bits = 10;
+      static const auto fraction_mask =
+        static_cast<uint32_t>(std::pow(2.0, fraction_n_bits)) - 1;
+      static const auto fraction_denominator = std::pow(2.0, fraction_n_bits);
+      const auto sign = value >> (exponent_n_bits + fraction_n_bits);
+      const auto exponent = (value >> fraction_n_bits) & exponent_mask;
+      const auto fraction = value & fraction_mask;
+      if (exponent == exponent_mask) {
+        if (sign == 0) {
+          return DBL2NUM(HUGE_VAL);
+        } else {
+          return DBL2NUM(-HUGE_VAL);
+        }
+      } else {
+        const auto implicit_fraction = (exponent == 0) ? 0 : 1;
+        return DBL2NUM(((sign == 0) ? 1 : -1) *
+                       std::pow(2.0, exponent - exponent_bias) *
+                       (implicit_fraction + fraction / fraction_denominator));
+      }
+    }
 
     inline VALUE convert(const arrow::FloatArray& array,
                          const int64_t i) {
@@ -320,8 +344,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -427,8 +450,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -530,8 +552,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -634,8 +655,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -665,25 +685,21 @@ namespace red_arrow {
   private:
     template <typename ArrayType>
     inline void convert_value(const ArrayType& array) {
-      auto result = rb_hash_new();
       if (array.IsNull(index_)) {
-        rb_hash_aset(result, field_name_, Qnil);
+        result_ = RUBY_Qnil;
       } else {
-        rb_hash_aset(result,
-                     field_name_,
-                     array_value_converter_->convert(array, index_));
+        result_ = array_value_converter_->convert(array, index_);
       }
-      result_ = result;
     }
 
-    uint8_t compute_field_index(const arrow::UnionArray& array,
-                                arrow::UnionType* type,
-                                const char* tag) {
+    int8_t compute_child_id(const arrow::UnionArray& array,
+                            arrow::UnionType* type,
+                            const char* tag) {
       const auto type_code = array.raw_type_codes()[index_];
       if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
-        const auto field_id = type->child_ids()[type_code];
-        if (field_id >= 0) {
-          return field_id;
+        const auto child_id = type->child_ids()[type_code];
+        if (child_id >= 0) {
+          return child_id;
         }
       }
       check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
@@ -695,36 +711,25 @@ namespace red_arrow {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-sparse-array]";
-      const auto index = compute_field_index(array, type, tag);
-      const auto field = type->field(index).get();
-      const auto& field_name = field->name();
-      const auto field_name_keep = field_name_;
-      field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto field_array = array.field(index).get();
+      const auto child_id = compute_child_id(array, type, tag);
+      const auto field_array = array.field(child_id).get();
       check_status(field_array->Accept(this), tag);
-      field_name_ = field_name_keep;
     }
 
     void convert_dense(const arrow::DenseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-dense-array]";
-      const auto index = compute_field_index(array, type, tag);
-      const auto field = type->field(index).get();
-      const auto& field_name = field->name();
-      const auto field_name_keep = field_name_;
-      field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto field_array = array.field(index);
+      const auto child_id = compute_child_id(array, type, tag);
+      const auto field_array = array.field(child_id);
       const auto index_keep = index_;
       index_ = array.value_offset(index_);
       check_status(field_array->Accept(this), tag);
       index_ = index_keep;
-      field_name_ = field_name_keep;
     }
 
     ArrayValueConverter* array_value_converter_;
     int64_t index_;
-    VALUE field_name_;
     VALUE result_;
   };
 
@@ -761,8 +766,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)

data/ext/arrow/extconf.rb

@@ -38,8 +38,6 @@ checking_for(checking_message("Homebrew")) do
   end
 end
 
-$CXXFLAGS += " -std=c++17 "
-
 unless required_pkg_config_package([
                                      "arrow",
                                      Arrow::Version::MAJOR,
@@ -77,4 +75,18 @@ end
   add_depend_package_path(name, source_dir, build_dir)
 end
 
+case RUBY_PLATFORM
+when /darwin/
+  symbols_in_external_bundles = [
+    "_rbgerr_gerror2exception",
+    "_rbgobj_instance_from_ruby_object",
+  ]
+  symbols_in_external_bundles.each do |symbol|
+    $DLDFLAGS << " -Wl,-U,#{symbol}"
+  end
+  mmacosx_version_min = "-mmacosx-version-min=10.14"
+  $CFLAGS << " #{mmacosx_version_min}"
+  $CXXFLAGS << " #{mmacosx_version_min}"
+end
+
 create_makefile("arrow")

data/ext/arrow/raw-records.cpp

@@ -84,8 +84,7 @@ namespace red_arrow {
       VISIT(UInt16)
       VISIT(UInt32)
       VISIT(UInt64)
-      // TODO
-      // VISIT(HalfFloat)
+      VISIT(HalfFloat)
       VISIT(Float)
       VISIT(Double)
       VISIT(Binary)

data/ext/arrow/values.cpp

@@ -65,8 +65,7 @@ namespace red_arrow {
       VISIT(UInt16)
       VISIT(UInt32)
       VISIT(UInt64)
-      // TODO
-      // VISIT(HalfFloat)
+      VISIT(HalfFloat)
       VISIT(Float)
       VISIT(Double)
       VISIT(Binary)

data/lib/arrow/array-computable.rb

@@ -29,6 +29,19 @@ module Arrow
       unique.values
     end
 
+    # Finds the index of the first occurrence of a given value.
+    #
+    # @param value [Object] The value to be compared.
+    #
+    # @return [Integer] The index of the first occurrence of a given
+    #   value on found, -1 on not found.
+    #
+    # @since 12.0.0
+    def index(value)
+      value = Scalar.resolve(value, value_data_type)
+      compute("index", options: {value: value}).value
+    end
+
     private
     def compute(name, options: nil)
       Function.find(name).execute([self], options).value

data/lib/arrow/array.rb

@@ -22,6 +22,7 @@ module Arrow
     include ArrayComputable
     include GenericFilterable
     include GenericTakeable
+    include InputReferable
 
     class << self
       def new(*args)
@@ -115,6 +116,10 @@ module Arrow
       self
     end
 
+    def to_arrow_chunked_array
+      ChunkedArray.new([self])
+    end
+
     alias_method :value_data_type_raw, :value_data_type
     def value_data_type
       @value_data_type ||= value_data_type_raw

data/lib/arrow/chunked-array.rb

@@ -22,6 +22,19 @@ module Arrow
     include ArrayComputable
     include GenericFilterable
     include GenericTakeable
+    include InputReferable
+
+    def to_arrow
+      self
+    end
+
+    def to_arrow_array
+      combine
+    end
+
+    def to_arrow_chunked_array
+      self
+    end
 
     alias_method :size, :n_rows
     unless method_defined?(:length)
@@ -30,7 +43,16 @@ module Arrow
 
     alias_method :chunks_raw, :chunks
     def chunks
-      @chunks ||= chunks_raw
+      @chunks ||= chunks_raw.tap do |_chunks|
+        _chunks.each do |chunk|
+          share_input(chunk)
+        end
+      end
+    end
+
+    alias_method :get_chunk_raw, :get_chunk
+    def get_chunk(i)
+      chunks[i]
     end
 
     def null?(i)

data/lib/arrow/column-containable.rb

@@ -143,5 +143,14 @@ module Arrow
         find_column(selector)
       end
     end
+
+    # Return column names in this object.
+    #
+    # @return [::Array<String>] column names.
+    #
+    # @since 11.0.0
+    def column_names
+      @column_names ||= columns.collect(&:name)
+    end
   end
 end

data/lib/arrow/column.rb

@@ -27,6 +27,7 @@ module Arrow
       @index = index
       @field = @container.schema[@index]
       @data = @container.get_column_data(@index)
+      @container.share_input(@data)
     end
 
     def name

data/lib/arrow/data-type.rb

@@ -199,5 +199,14 @@ module Arrow
       args.unshift(self) unless builder_class.buildable?(args)
       builder_class.build(*args)
     end
+
+    # @return [Arrow::Scalar} A corresponding {Arrow::Scalar} class
+    #   for this data type.
+    #
+    # @since 12.0.0
+    def scalar_class
+      base_name = self.class.name.gsub(/DataType\z/, "")
+      ::Arrow.const_get("#{base_name}Scalar")
+    end
   end
 end

data/lib/arrow/dense-union-array-builder.rb

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DenseUnionArrayBuilder
+    alias_method :append_value_raw, :append_value
+
+    # @overload append_value
+    #
+    #   Starts appending an union record. You need to append values of
+    #   fields.
+    #
+    # @overload append_value(value)
+    #
+    #   Appends an union record including values of fields.
+    #
+    #   @param value [nil, Hash] The union record value.
+    #
+    #     If this is `nil`, the union record is null.
+    #
+    #     If this is `Hash`, it's  values of fields.
+    #
+    # @since 12.0.0
+    def append_value(value)
+      if value.nil?
+        append_null
+      else
+        key = value.keys[0]
+        child_info = child_infos[key]
+        append_value_raw(child_info[:id])
+        child_info[:builder].append(value.values[0])
+      end
+    end
+  end
+end

data/lib/arrow/dense-union-array.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DenseUnionArray
+    def get_value(i)
+      child_id = get_child_id(i)
+      field = get_field(child_id)
+      field[get_value_offset(i)]
+    end
+  end
+end

data/lib/arrow/half-float-array-builder.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloatArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      case value
+      when Float
+        HalfFloat.new(value).to_uint16
+      when HalfFloat
+        value.to_uint16
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/half-float-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloatArray
+    def get_value(i)
+      HalfFloat.new(get_raw_value(i)).to_f
+    end
+  end
+end

data/lib/arrow/half-float.rb

@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class HalfFloat
+    MAX = 65504
+    MIN = -65504
+    EXPONENT_N_BITS = 5
+    EXPONENT_MASK = (2 ** EXPONENT_N_BITS) - 1
+    EXPONENT_BIAS = 15
+    FRACTION_N_BITS = 10
+    FRACTION_MASK = (2 ** FRACTION_N_BITS) - 1
+    FRACTION_DENOMINATOR = 2.0 ** FRACTION_N_BITS
+
+    attr_reader :sign
+    attr_reader :exponent
+    attr_reader :fraction
+    def initialize(*args)
+      n_args = args.size
+      case n_args
+      when 1
+        if args[0].is_a?(Float)
+          @sign, @exponent, @fraction = deconstruct_float(args[0])
+        else
+          @sign, @exponent, @fraction = deconstruct_uint16(args[0])
+        end
+      when 3
+        @sign, @exponent, @fraction = *args
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 1 or 3)"
+        raise ArgumentError, message
+      end
+    end
+
+    def to_f
+      if @exponent == EXPONENT_MASK
+        if @sign.zero?
+          Float::INFINITY
+        else
+          -Float::INFINITY
+        end
+      else
+        if @exponent.zero?
+          implicit_fraction = 0
+        else
+          implicit_fraction = 1
+        end
+        ((-1) ** @sign) *
+          (2 ** (@exponent - EXPONENT_BIAS)) *
+          (implicit_fraction + @fraction / FRACTION_DENOMINATOR)
+      end
+    end
+
+    def to_uint16
+      (@sign << (EXPONENT_N_BITS + FRACTION_N_BITS)) ^
+        (@exponent << FRACTION_N_BITS) ^
+        @fraction
+    end
+
+    def pack
+      [to_uint16].pack("S")
+    end
+
+    private
+    def deconstruct_float(float)
+      if float > MAX
+        float = Float::INFINITY
+      elsif float < MIN
+        float = -Float::INFINITY
+      end
+      is_infinite = float.infinite?
+      if is_infinite
+        sign = (is_infinite == 1) ? 0 : 1
+        exponent = EXPONENT_MASK
+        fraction = 0
+      elsif float.zero?
+        sign = 0
+        exponent = 0
+        fraction = 0
+      else
+        sign = (float.positive? ? 0 : 1)
+        float_abs = float.abs
+        1.upto(EXPONENT_MASK) do |e|
+          next_exponent_value = 2 ** (e + 1 - EXPONENT_BIAS)
+          next if float_abs > next_exponent_value
+          exponent = e
+          exponent_value = 2 ** (e - EXPONENT_BIAS)
+          fraction =
+            ((float_abs / exponent_value - 1) * FRACTION_DENOMINATOR).round
+          break
+        end
+      end
+      [sign, exponent, fraction]
+    end
+
+    def deconstruct_uint16(uint16)
+      # | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
+      sign = (uint16 >> (EXPONENT_N_BITS + FRACTION_N_BITS))
+      exponent = ((uint16 >> FRACTION_N_BITS) & EXPONENT_MASK)
+      fraction = (uint16 & FRACTION_MASK)
+      [sign, exponent, fraction]
+    end
+  end
+end

data/lib/arrow/input-referable.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module InputReferable
+    def refer_input(input)
+      @input = input
+    end
+
+    def share_input(other)
+      return unless defined?(@input)
+      other.refer_input(@input)
+    end
+  end
+end