red-arrow 0.17.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 661c88599c8cb32f10fbea1e03994a861bb0a14da3617a0124c614428dc8e016
-  data.tar.gz: b478f0588467345bdce0da3c8a10453767bb0ce9cf64586bbd38411363f4e423
+  metadata.gz: f048aad4d5cc38373c1acc3976ba7012292c2b21315be7334704b6abbd4076fa
+  data.tar.gz: 426aac582a879286bd8f6ae632f4851bcc00025a08032142f07073fd4c665c62
 SHA512:
-  metadata.gz: e2c72551f2211db790cb0627754290a5d1ddf6583d0b14b830ee5922786a5096d1ad0e2b78849f8213b2164a4c2e6803fd6317d65d5a3f36cbf71b3c660bbd51
-  data.tar.gz: b1271385c4e346627dd58340a369742307fb432ce29a0f2acab501c385c1677c50ed885e3528f5d895bc8a35d5e5a99ddc2ee79357c902cd2236186d6da647d5
+  metadata.gz: 5f7ab5ce2884d205b7d498db7f74d50c9c6228a4c27de8638fb921535c71fed28aa69ffdcb55bc27300e8b5995d5f31b0e78a29b2bd0513fcbf60d1514a58f79
+  data.tar.gz: b6d28706fb0b4845c2b134e22f4988199e8714bf41e24a73d02b38a6161142ad04c7ebf30185791afb3c3a0b7a02701e92193b6ae55ff1e78089d60f3ca91c9a

data/ext/arrow/converters.hpp

@@ -212,7 +212,17 @@ namespace red_arrow {
 
     inline VALUE convert(const arrow::Decimal128Array& array,
                          const int64_t i) {
-      decimal_buffer_ = array.FormatValue(i);
+      return convert_decimal(std::move(array.FormatValue(i)));
+    }
+
+    inline VALUE convert(const arrow::Decimal256Array& array,
+                         const int64_t i) {
+      return convert_decimal(std::move(array.FormatValue(i)));
+    }
+
+  private:
+    inline VALUE convert_decimal(std::string&& value) {
+      decimal_buffer_ = value;
       return rb_funcall(rb_cObject,
                         id_BigDecimal,
                         1,
@@ -221,7 +231,6 @@ namespace red_arrow {
                                        rb_ascii8bit_encoding()));
     }
 
-  private:
     std::string decimal_buffer_;
     ListArrayValueConverter* list_array_value_converter_;
     StructArrayValueConverter* struct_array_value_converter_;
@@ -285,9 +294,11 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
+    VISIT(Decimal256)
     // TODO
     // VISIT(Extension)
 
@@ -339,9 +350,9 @@ namespace red_arrow {
       index_ = index;
       result_ = rb_hash_new();
       const auto struct_type = array.struct_type();
-      const auto n = struct_type->num_children();
+      const auto n = struct_type->num_fields();
       for (int i = 0; i < n; ++i) {
-        const auto field_type = struct_type->child(i).get();
+        const auto field_type = struct_type->field(i).get();
         const auto& field_name = field_type->name();
         auto key_keep = key_;
         key_ = rb_utf8_str_new(field_name.data(), field_name.length());
@@ -388,9 +399,11 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
+    VISIT(Decimal256)
     // TODO
     // VISIT(Extension)
 
@@ -432,10 +445,10 @@ namespace red_arrow {
       index_ = index;
       switch (array.mode()) {
       case arrow::UnionMode::SPARSE:
-        convert_sparse(array);
+        convert_sparse(static_cast<const arrow::SparseUnionArray&>(array));
         break;
       case arrow::UnionMode::DENSE:
-        convert_dense(array);
+        convert_dense(static_cast<const arrow::DenseUnionArray&>(array));
         break;
       default:
         rb_raise(rb_eArgError, "Invalid union mode");
@@ -479,9 +492,11 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
+    VISIT(Decimal256)
     // TODO
     // VISIT(Extension)
 
@@ -501,14 +516,14 @@ namespace red_arrow {
       result_ = result;
     }
 
-    uint8_t compute_child_index(const arrow::UnionArray& array,
+    uint8_t compute_field_index(const arrow::UnionArray& array,
                                 arrow::UnionType* type,
                                 const char* tag) {
       const auto type_code = array.raw_type_codes()[index_];
       if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
-        const auto child_id = type->child_ids()[type_code];
-        if (child_id >= 0) {
-          return child_id;
+        const auto field_id = type->child_ids()[type_code];
+        if (field_id >= 0) {
+          return field_id;
         }
       }
       check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
@@ -516,33 +531,33 @@ namespace red_arrow {
       return 0;
     }
 
-    void convert_sparse(const arrow::UnionArray& array) {
+    void convert_sparse(const arrow::SparseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-sparse-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index).get();
-      check_status(child_array->Accept(this), tag);
+      const auto field_array = array.field(index).get();
+      check_status(field_array->Accept(this), tag);
       field_name_ = field_name_keep;
     }
 
-    void convert_dense(const arrow::UnionArray& array) {
+    void convert_dense(const arrow::DenseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-dense-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index);
+      const auto field_array = array.field(index);
       const auto index_keep = index_;
       index_ = array.value_offset(index_);
-      check_status(child_array->Accept(this), tag);
+      check_status(field_array->Accept(this), tag);
       index_ = index_keep;
       field_name_ = field_name_keep;
     }
@@ -557,30 +572,58 @@ namespace red_arrow {
   public:
     explicit DictionaryArrayValueConverter(ArrayValueConverter* converter)
       : array_value_converter_(converter),
-        index_(0),
+        value_index_(0),
         result_(Qnil) {
     }
 
     VALUE convert(const arrow::DictionaryArray& array,
                   const int64_t index) {
-      index_ = index;
-      auto indices = array.indices().get();
-      check_status(indices->Accept(this),
+      value_index_ = array.GetValueIndex(index);
+      auto dictionary = array.dictionary().get();
+      check_status(dictionary->Accept(this),
                    "[raw-records][dictionary-array]");
       return result_;
     }
 
-    // TODO: Convert to real value.
 #define VISIT(TYPE)                                                     \
     arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
-      result_ = convert_value(array, index_);                           \
+      result_ = convert_value(array, value_index_);                     \
       return arrow::Status::OK();                                       \
       }
 
+    VISIT(Null)
+    VISIT(Boolean)
     VISIT(Int8)
     VISIT(Int16)
     VISIT(Int32)
     VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    // TODO
+    // VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    // TODO
+    // VISIT(Interval)
+    VISIT(List)
+    VISIT(Struct)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    VISIT(Decimal256)
+    // TODO
+    // VISIT(Extension)
 
 #undef VISIT
 
@@ -592,7 +635,7 @@ namespace red_arrow {
     }
 
     ArrayValueConverter* array_value_converter_;
-    int64_t index_;
+    int64_t value_index_;
     VALUE result_;
   };
 

data/ext/arrow/extconf.rb

@@ -16,7 +16,8 @@
 # under the License.
 
 require "extpp"
-require "mkmf-gnome2"
+require "mkmf-gnome"
+require_relative "../../lib/arrow/version"
 
 arrow_pkg_config_path = ENV["ARROW_PKG_CONFIG_PATH"]
 if arrow_pkg_config_path
@@ -24,7 +25,12 @@ if arrow_pkg_config_path
   ENV["PKG_CONFIG_PATH"] = pkg_config_paths.join(File::PATH_SEPARATOR)
 end
 
-unless required_pkg_config_package("arrow",
+unless required_pkg_config_package([
+                                     "arrow",
+                                     Arrow::Version::MAJOR,
+                                     Arrow::Version::MINOR,
+                                     Arrow::Version::MICRO,
+                                   ],
                                    debian: "libarrow-dev",
                                    redhat: "arrow-devel",
                                    homebrew: "apache-arrow",
@@ -32,7 +38,12 @@ unless required_pkg_config_package("arrow",
   exit(false)
 end
 
-unless required_pkg_config_package("arrow-glib",
+unless required_pkg_config_package([
+                                     "arrow-glib",
+                                     Arrow::Version::MAJOR,
+                                     Arrow::Version::MINOR,
+                                     Arrow::Version::MICRO,
+                                   ],
                                    debian: "libarrow-glib-dev",
                                    redhat: "arrow-glib-devel",
                                    homebrew: "apache-arrow-glib",

data/ext/arrow/raw-records.cpp

@@ -100,9 +100,11 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
+      VISIT(Decimal256)
       // TODO
       // VISIT(Extension)
 

data/ext/arrow/values.cpp

@@ -81,9 +81,11 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
+      VISIT(Decimal256)
       // TODO
       // VISIT(Extension)
 

data/lib/arrow/array-builder.rb

@@ -115,6 +115,17 @@ module Arrow
             builder: Date32ArrayBuilder.new,
             detected: true,
           }
+        when BigDecimal
+          if value.to_arrow.is_a?(Decimal128)
+            {
+              builder: Decimal128ArrayBuilder.new,
+            }
+          else
+            {
+              builder: Decimal256ArrayBuilder.new,
+              detected: true,
+            }
+          end
         when ::Array
           sub_builder_info = nil
           value.each do |sub_value|
@@ -194,11 +205,5 @@ module Arrow
         end
       end
     end
-
-    def append_nulls(n)
-      n.times do
-        append_null
-      end
-    end
   end
 end

data/lib/arrow/array.rb

@@ -100,5 +100,123 @@ module Arrow
         is_in_raw(values)
       end
     end
+
+    # @api private
+    alias_method :concatenate_raw, :concatenate
+    # Concatenates the given other arrays to the array.
+    #
+    # @param other_arrays [::Array, Arrow::Array] The arrays to be
+    #   concatenated.
+    #
+    #   Each other array is processed by {#resolve} before they're
+    #   concatenated.
+    #
+    # @example Raw Ruby Array
+    #   array = Arrow::Int32Array.new([1])
+    #   array.concatenate([2, 3], [4]) # => Arrow::Int32Array.new([1, 2, 3, 4])
+    #
+    # @example Arrow::Array
+    #   array = Arrow::Int32Array.new([1])
+    #   array.concatenate(Arrow::Int32Array.new([2, 3]),
+    #                     Arrow::Int8Array.new([4])) # => Arrow::Int32Array.new([1, 2, 3, 4])
+    #
+    # @since 4.0.0
+    def concatenate(*other_arrays)
+      other_arrays = other_arrays.collect do |other_array|
+        resolve(other_array)
+      end
+      concatenate_raw(other_arrays)
+    end
+
+    # Concatenates the given other array to the array.
+    #
+    # If you have multiple arrays to be concatenated, you should use
+    # {#concatenate} to concatenate multiple arrays at once.
+    #
+    # @param other_array [::Array, Arrow::Array] The array to be concatenated.
+    #
+    #   `@other_array` is processed by {#resolve} before it's
+    #   concatenated.
+    #
+    # @example Raw Ruby Array
+    #   Arrow::Int32Array.new([1]) + [2, 3] # => Arrow::Int32Array.new([1, 2, 3])
+    #
+    # @example Arrow::Array
+    #   Arrow::Int32Array.new([1]) +
+    #     Arrow::Int32Array.new([2, 3]) # => Arrow::Int32Array.new([1, 2, 3])
+    #
+    # @since 4.0.0
+    def +(other_array)
+      concatenate(other_array)
+    end
+
+    # Ensures returning the same data type array from the given array.
+    #
+    # @return [Arrow::Array]
+    #
+    # @overload resolve(other_raw_array)
+    #
+    #   @param other_raw_array [::Array] A raw Ruby Array. A new Arrow::Array
+    #     is built by `self.class.new`.
+    #
+    #   @example Raw Ruby Array
+    #     int32_array = Arrow::Int32Array.new([1])
+    #     other_array = int32_array.resolve([2, 3, 4])
+    #     other_array # => Arrow::Int32Array.new([2, 3, 4])
+    #
+    # @overload resolve(other_array)
+    #
+    #   @param other_array [Arrow::Array] Another Arrow::Array.
+    #
+    #     If the given other array is an same data type array of
+    #     `self`, the given other array is returned as-is.
+    #
+    #     If the given other array isn't an same data type array of
+    #     `self`, the given other array is casted.
+    #
+    #   @example Same data type
+    #     int32_array = Arrow::Int32Array.new([1])
+    #     other_int32_array = Arrow::Int32Array.new([2, 3, 4])
+    #     other_array = int32_array.resolve(other_int32_array)
+    #     other_array.object_id == other_int32_array.object_id
+    #
+    #   @example Other data type
+    #     int32_array = Arrow::Int32Array.new([1])
+    #     other_int8_array = Arrow::Int8Array.new([2, 3, 4])
+    #     other_array = int32_array.resolve(other_int32_array)
+    #     other_array #=> Arrow::Int32Array.new([2, 3, 4])
+    #
+    # @since 4.0.0
+    def resolve(other_array)
+      if other_array.is_a?(::Array)
+        builder_class = self.class.builder_class
+        if builder_class.nil?
+          message =
+            "[array][resolve] can't build #{value_data_type} array " +
+            "from raw Ruby Array"
+          raise ArgumentError, message
+        end
+        if builder_class.buildable?([other_array])
+          other_array = builder_class.build(other_array)
+        elsif builder_class.buildable?([value_data_type, other_array])
+          other_array = builder_class.build(value_data_type, other_array)
+        else
+          message =
+            "[array][resolve] need to implement " +
+            "a feature that building #{value_data_type} array " +
+            "from raw Ruby Array"
+          raise NotImpelemented, message
+        end
+        other_array
+      elsif other_array.respond_to?(:value_data_type)
+        return other_array if value_data_type == other_array.value_data_type
+        other_array.cast(value_data_type)
+      else
+        message =
+          "[array][resolve] can't build #{value_data_type} array: " +
+          "#{other_array.inspect}"
+        raise ArgumentError, message
+      end
+    end
   end
 end