red-arrow 8.0.0 → 24.0.0

data/ext/arrow/converters.hpp

@@ -28,6 +28,8 @@
 
 namespace red_arrow {
   class ListArrayValueConverter;
+  class LargeListArrayValueConverter;
+  class FixedSizeListArrayValueConverter;
   class StructArrayValueConverter;
   class MapArrayValueConverter;
   class UnionArrayValueConverter;
@@ -38,6 +40,8 @@ namespace red_arrow {
     ArrayValueConverter()
       : decimal_buffer_(),
         list_array_value_converter_(nullptr),
+        large_list_array_value_converter_(nullptr),
+        fixed_size_list_array_value_converter_(nullptr),
         struct_array_value_converter_(nullptr),
         map_array_value_converter_(nullptr),
         union_array_value_converter_(nullptr),
@@ -45,11 +49,15 @@ namespace red_arrow {
     }
 
     inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter,
+                                         LargeListArrayValueConverter* large_list_array_value_converter,
+                                         FixedSizeListArrayValueConverter* fixed_size_list_array_value_converter,
                                          StructArrayValueConverter* struct_array_value_converter,
                                          MapArrayValueConverter* map_array_value_converter,
                                          UnionArrayValueConverter* union_array_value_converter,
                                          DictionaryArrayValueConverter* dictionary_array_value_converter) {
       list_array_value_converter_ = list_array_value_converter;
+      large_list_array_value_converter_ = large_list_array_value_converter;
+      fixed_size_list_array_value_converter_ = fixed_size_list_array_value_converter;
       struct_array_value_converter_ = struct_array_value_converter;
       map_array_value_converter_ = map_array_value_converter;
       union_array_value_converter_ = union_array_value_converter;
@@ -106,10 +114,34 @@ namespace red_arrow {
       return ULL2NUM(array.Value(i));
     }
 
-    // TODO
-    // inline VALUE convert(const arrow::HalfFloatArray& array,
-    //                      const int64_t i) {
-    // }
+    inline VALUE convert(const arrow::HalfFloatArray& array,
+                         const int64_t i) {
+      const auto value = array.Value(i);
+      // | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
+      constexpr auto exponent_n_bits = 5;
+      static const auto exponent_mask =
+        static_cast<uint32_t>(std::pow(2.0, exponent_n_bits) - 1);
+      constexpr auto exponent_bias = 15;
+      constexpr auto fraction_n_bits = 10;
+      static const auto fraction_mask =
+        static_cast<uint32_t>(std::pow(2.0, fraction_n_bits)) - 1;
+      static const auto fraction_denominator = std::pow(2.0, fraction_n_bits);
+      const auto sign = value >> (exponent_n_bits + fraction_n_bits);
+      const auto exponent = (value >> fraction_n_bits) & exponent_mask;
+      const auto fraction = value & fraction_mask;
+      if (exponent == exponent_mask) {
+        if (sign == 0) {
+          return DBL2NUM(HUGE_VAL);
+        } else {
+          return DBL2NUM(-HUGE_VAL);
+        }
+      } else {
+        const auto implicit_fraction = (exponent == 0) ? 0 : 1;
+        return DBL2NUM(((sign == 0) ? 1 : -1) *
+                       std::pow(2.0, exponent - exponent_bias) *
+                       (implicit_fraction + fraction / fraction_denominator));
+      }
+    }
 
     inline VALUE convert(const arrow::FloatArray& array,
                          const int64_t i) {
@@ -125,7 +157,15 @@ namespace red_arrow {
                          const int64_t i) {
       int32_t length;
       const auto value = array.GetValue(i, &length);
-      // TODO: encoding support
+      return rb_enc_str_new(reinterpret_cast<const char*>(value),
+                            length,
+                            rb_ascii8bit_encoding());
+    }
+
+    inline VALUE convert(const arrow::LargeBinaryArray& array,
+                         const int64_t i) {
+      int64_t length;
+      const auto value = array.GetValue(i, &length);
       return rb_enc_str_new(reinterpret_cast<const char*>(value),
                             length,
                             rb_ascii8bit_encoding());
@@ -139,6 +179,14 @@ namespace red_arrow {
                              length);
     }
 
+    inline VALUE convert(const arrow::LargeStringArray& array,
+                         const int64_t i) {
+      int64_t length;
+      const auto value = array.GetValue(i, &length);
+      return rb_utf8_str_new(reinterpret_cast<const char*>(value),
+                             length);
+    }
+
     inline VALUE convert(const arrow::FixedSizeBinaryArray& array,
                          const int64_t i) {
       return rb_enc_str_new(reinterpret_cast<const char*>(array.Value(i)),
@@ -197,11 +245,6 @@ namespace red_arrow {
       return rb_time_num_new(sec, Qnil);
     }
 
-    // TODO
-    // inline VALUE convert(const arrow::IntervalArray& array,
-    //                      const int64_t i) {
-    // };
-
     inline VALUE convert(const arrow::MonthIntervalArray& array,
                          const int64_t i) {
       return INT2NUM(array.Value(i));
@@ -236,9 +279,20 @@ namespace red_arrow {
       return value;
     }
 
+    inline VALUE convert(const arrow::DurationArray& array,
+                         const int64_t i) {
+      return LL2NUM(array.Value(i));
+    }
+
     VALUE convert(const arrow::ListArray& array,
                   const int64_t i);
 
+    VALUE convert(const arrow::LargeListArray& array,
+                  const int64_t i);
+
+    VALUE convert(const arrow::FixedSizeListArray& array,
+                  const int64_t i);
+
     VALUE convert(const arrow::StructArray& array,
                   const int64_t i);
 
@@ -274,6 +328,8 @@ namespace red_arrow {
 
     std::string decimal_buffer_;
     ListArrayValueConverter* list_array_value_converter_;
+    LargeListArrayValueConverter* large_list_array_value_converter_;
+    FixedSizeListArrayValueConverter* fixed_size_list_array_value_converter_;
     StructArrayValueConverter* struct_array_value_converter_;
     MapArrayValueConverter* map_array_value_converter_;
     UnionArrayValueConverter* union_array_value_converter_;
@@ -320,8 +376,209 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
+    VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
+    VISIT(List)
+    VISIT(LargeList)
+    VISIT(FixedSizeList)
+    VISIT(Struct)
+    VISIT(Map)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    VISIT(Decimal256)
+    // TODO
+    // VISIT(Extension)
+
+#undef VISIT
+
+  private:
+    template <typename ArrayType>
+    inline VALUE convert_value(const ArrayType& array,
+                               const int64_t i) {
+      return array_value_converter_->convert(array, i);
+    }
+
+    template <typename ArrayType>
+    arrow::Status visit_value(const ArrayType& array) {
+      if (array.null_count() > 0) {
+        for (int64_t i = 0; i < length_; ++i) {
+          auto value = Qnil;
+          if (!array.IsNull(i + offset_)) {
+            value = convert_value(array, i + offset_);
+          }
+          rb_ary_push(result_, value);
+        }
+      } else {
+        for (int64_t i = 0; i < length_; ++i) {
+          rb_ary_push(result_, convert_value(array, i + offset_));
+        }
+      }
+      return arrow::Status::OK();
+    }
+
+    ArrayValueConverter* array_value_converter_;
+    int32_t offset_;
+    int32_t length_;
+    VALUE result_;
+  };
+
+  class LargeListArrayValueConverter : public arrow::ArrayVisitor {
+  public:
+    explicit LargeListArrayValueConverter(ArrayValueConverter* converter)
+      : array_value_converter_(converter),
+        offset_(0),
+        length_(0),
+        result_(Qnil) {}
+
+    VALUE convert(const arrow::LargeListArray& array, const int64_t index) {
+      auto values = array.values().get();
+      auto offset_keep = offset_;
+      auto length_keep = length_;
+      offset_ = array.value_offset(index);
+      length_ = array.value_length(index);
+      auto result_keep = result_;
+      result_ = rb_ary_new_capa(length_);
+      check_status(values->Accept(this),
+                   "[raw-records][large-list-array]");
+      offset_ = offset_keep;
+      length_ = length_keep;
+      auto result_return = result_;
+      result_ = result_keep;
+      return result_return;
+    }
+
+#define VISIT(TYPE)                                                     \
+    arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
+      return visit_value(array);                                        \
+    }
+
+    VISIT(Null)
+    VISIT(Boolean)
+    VISIT(Int8)
+    VISIT(Int16)
+    VISIT(Int32)
+    VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
+    VISIT(List)
+    VISIT(LargeList)
+    VISIT(FixedSizeList)
+    VISIT(Struct)
+    VISIT(Map)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    VISIT(Decimal256)
     // TODO
-    // VISIT(HalfFloat)
+    // VISIT(Extension)
+
+#undef VISIT
+
+  private:
+    template <typename ArrayType>
+    inline VALUE convert_value(const ArrayType& array,
+                               const int64_t i) {
+      return array_value_converter_->convert(array, i);
+    }
+
+    template <typename ArrayType>
+    arrow::Status visit_value(const ArrayType& array) {
+      if (array.null_count() > 0) {
+        for (int64_t i = 0; i < length_; ++i) {
+          auto value = Qnil;
+          if (!array.IsNull(i + offset_)) {
+            value = convert_value(array, i + offset_);
+          }
+          rb_ary_push(result_, value);
+        }
+      } else {
+        for (int64_t i = 0; i < length_; ++i) {
+          rb_ary_push(result_, convert_value(array, i + offset_));
+        }
+      }
+      return arrow::Status::OK();
+    }
+
+    ArrayValueConverter* array_value_converter_;
+    int32_t offset_;
+    int32_t length_;
+    VALUE result_;
+  };
+
+  class FixedSizeListArrayValueConverter : public arrow::ArrayVisitor {
+  public:
+    explicit FixedSizeListArrayValueConverter(ArrayValueConverter* converter)
+      : array_value_converter_(converter),
+        offset_(0),
+        length_(0),
+        result_(Qnil) {}
+
+    VALUE convert(const arrow::FixedSizeListArray& array, const int64_t index) {
+      auto values = array.values().get();
+      auto offset_keep = offset_;
+      auto length_keep = length_;
+      offset_ = array.value_offset(index);
+      length_ = array.value_length(index);
+      auto result_keep = result_;
+      result_ = rb_ary_new_capa(length_);
+      check_status(values->Accept(this),
+                   "[raw-records][fixed-size-list-array]");
+      offset_ = offset_keep;
+      length_ = length_keep;
+      auto result_return = result_;
+      result_ = result_keep;
+      return result_return;
+    }
+
+#define VISIT(TYPE)                                                     \
+    arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
+      return visit_value(array);                                        \
+    }
+
+    VISIT(Null)
+    VISIT(Boolean)
+    VISIT(Int8)
+    VISIT(Int16)
+    VISIT(Int32)
+    VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -335,7 +592,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
+    VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -427,8 +687,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -442,7 +701,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
+    VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -530,8 +792,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -545,7 +806,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
+    VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -634,8 +898,7 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
@@ -649,7 +912,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
+    VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -665,25 +931,21 @@ namespace red_arrow {
   private:
     template <typename ArrayType>
     inline void convert_value(const ArrayType& array) {
-      auto result = rb_hash_new();
       if (array.IsNull(index_)) {
-        rb_hash_aset(result, field_name_, Qnil);
+        result_ = RUBY_Qnil;
       } else {
-        rb_hash_aset(result,
-                     field_name_,
-                     array_value_converter_->convert(array, index_));
+        result_ = array_value_converter_->convert(array, index_);
       }
-      result_ = result;
     }
 
-    uint8_t compute_field_index(const arrow::UnionArray& array,
-                                arrow::UnionType* type,
-                                const char* tag) {
+    int8_t compute_child_id(const arrow::UnionArray& array,
+                            arrow::UnionType* type,
+                            const char* tag) {
       const auto type_code = array.raw_type_codes()[index_];
       if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
-        const auto field_id = type->child_ids()[type_code];
-        if (field_id >= 0) {
-          return field_id;
+        const auto child_id = type->child_ids()[type_code];
+        if (child_id >= 0) {
+          return child_id;
         }
       }
       check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
@@ -695,36 +957,25 @@ namespace red_arrow {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-sparse-array]";
-      const auto index = compute_field_index(array, type, tag);
-      const auto field = type->field(index).get();
-      const auto& field_name = field->name();
-      const auto field_name_keep = field_name_;
-      field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto field_array = array.field(index).get();
+      const auto child_id = compute_child_id(array, type, tag);
+      const auto field_array = array.field(child_id).get();
       check_status(field_array->Accept(this), tag);
-      field_name_ = field_name_keep;
     }
 
     void convert_dense(const arrow::DenseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-dense-array]";
-      const auto index = compute_field_index(array, type, tag);
-      const auto field = type->field(index).get();
-      const auto& field_name = field->name();
-      const auto field_name_keep = field_name_;
-      field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto field_array = array.field(index);
+      const auto child_id = compute_child_id(array, type, tag);
+      const auto field_array = array.field(child_id);
       const auto index_keep = index_;
       index_ = array.value_offset(index_);
       check_status(field_array->Accept(this), tag);
       index_ = index_keep;
-      field_name_ = field_name_keep;
     }
 
     ArrayValueConverter* array_value_converter_;
     int64_t index_;
-    VALUE field_name_;
     VALUE result_;
   };
 
@@ -761,19 +1012,26 @@ namespace red_arrow {
     VISIT(UInt16)
     VISIT(UInt32)
     VISIT(UInt64)
-    // TODO
-    // VISIT(HalfFloat)
+    VISIT(HalfFloat)
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
+    VISIT(LargeBinary)
     VISIT(String)
+    VISIT(LargeString)
     VISIT(FixedSizeBinary)
     VISIT(Date32)
     VISIT(Date64)
     VISIT(Time32)
     VISIT(Time64)
     VISIT(Timestamp)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
+    VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -803,12 +1061,16 @@ namespace red_arrow {
     explicit Converter()
       : array_value_converter_(),
         list_array_value_converter_(&array_value_converter_),
+        large_list_array_value_converter_(&array_value_converter_),
+        fixed_size_list_array_value_converter_(&array_value_converter_),
         struct_array_value_converter_(&array_value_converter_),
         map_array_value_converter_(&array_value_converter_),
         union_array_value_converter_(&array_value_converter_),
         dictionary_array_value_converter_(&array_value_converter_) {
       array_value_converter_.
         set_sub_value_converters(&list_array_value_converter_,
+                                 &large_list_array_value_converter_,
+                                 &fixed_size_list_array_value_converter_,
                                  &struct_array_value_converter_,
                                  &map_array_value_converter_,
                                  &union_array_value_converter_,
@@ -823,6 +1085,8 @@ namespace red_arrow {
 
     ArrayValueConverter array_value_converter_;
     ListArrayValueConverter list_array_value_converter_;
+    LargeListArrayValueConverter large_list_array_value_converter_;
+    FixedSizeListArrayValueConverter fixed_size_list_array_value_converter_;
     StructArrayValueConverter struct_array_value_converter_;
     MapArrayValueConverter map_array_value_converter_;
     UnionArrayValueConverter union_array_value_converter_;

data/ext/arrow/extconf.rb

@@ -38,32 +38,37 @@ checking_for(checking_message("Homebrew")) do
   end
 end
 
-unless required_pkg_config_package([
-                                     "arrow",
-                                     Arrow::Version::MAJOR,
-                                     Arrow::Version::MINOR,
-                                     Arrow::Version::MICRO,
-                                   ],
-                                   debian: "libarrow-dev",
-                                   redhat: "arrow-devel",
-                                   homebrew: "apache-arrow",
-                                   msys2: "arrow")
-  exit(false)
+unless PKGConfig.have_package("arrow", Arrow::Version::MAJOR)
+  raise <<-MESSAGE
+Apache Arrow C++ >= #{Arrow::Version::MAJOR} isn't found.
+You can install it automatically by enabling rubygems-requirements-system.
+See https://github.com/ruby-gnome/rubygems-requirements-system/ how to enable it.
+  MESSAGE
 end
 
-unless required_pkg_config_package([
-                                     "arrow-glib",
-                                     Arrow::Version::MAJOR,
-                                     Arrow::Version::MINOR,
-                                     Arrow::Version::MICRO,
-                                   ],
-                                   debian: "libarrow-glib-dev",
-                                   redhat: "arrow-glib-devel",
-                                   homebrew: "apache-arrow-glib",
-                                   msys2: "arrow")
-  exit(false)
+unless PKGConfig.have_package("arrow-glib",
+                              Arrow::Version::MAJOR,
+                              Arrow::Version::MINOR,
+                              Arrow::Version::MICRO)
+  version = [
+    Arrow::Version::MAJOR,
+    Arrow::Version::MINOR,
+    Arrow::Version::MICRO,
+  ].join(".")
+  raise <<-MESSAGE
+Apache Arrow GLib >= #{version} isn't found.
+You can install it automatically by enabling rubygems-requirements-system.
+See https://github.com/ruby-gnome/rubygems-requirements-system/ how to enable it.
+  MESSAGE
 end
 
+# Old re2.pc (e.g. re2.pc on Ubuntu 20.04) may add -std=c++11. It
+# causes a build error because Apache Arrow C++ requires C++17 or
+# later.
+#
+# We can remove this when we drop support for Ubuntu 20.04.
+$CXXFLAGS.gsub!("-std=c++11", "")
+
 [
   ["glib2", "ext/glib2"],
 ].each do |name, relative_source_dir|
@@ -73,4 +78,18 @@ end
   add_depend_package_path(name, source_dir, build_dir)
 end
 
+case RUBY_PLATFORM
+when /darwin/
+  symbols_in_external_bundles = [
+    "_rbgerr_gerror2exception",
+    "_rbgobj_instance_from_ruby_object",
+  ]
+  symbols_in_external_bundles.each do |symbol|
+    $DLDFLAGS << " -Wl,-U,#{symbol}"
+  end
+  mmacosx_version_min = "-mmacosx-version-min=12.0"
+  $CFLAGS << " #{mmacosx_version_min}"
+  $CXXFLAGS << " #{mmacosx_version_min}"
+end
+
 create_makefile("arrow")