red-arrow 0.17.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba4a00fdcc27e2e75985b0d0da37eff70530aef9431d5234da72517313ebad55
-  data.tar.gz: 50fb004195b273ec5cbf3e9fa7c92fcb3a7755f010e2ddd469d84c7c27a104df
+  metadata.gz: a70c1505c294a8f74c992ec0f2ab5d651647e45dd178db8523f5b5c01e64a541
+  data.tar.gz: dad979599033104a25d5be3d05c57e552c803f9c8002d66b757866c425937ce9
 SHA512:
-  metadata.gz: 0d0e852a7d0ea89c03ac752e779a1bd5b0ec6a81a10b6b457d66857f2dacb5e445dc41f5254e8985f3d7b8ec6328cedea92bd22758b84a7e935278374e51fcd3
-  data.tar.gz: 4aa5f3c817f2ef1a475ed7f272f37f495d4f9c66f07a98381a04a6b16ee9b803904f2c0982048003b0bbe6038369d33b3632b6cb437d849d25179b504233e142
+  metadata.gz: b2363ba6468985a3d237cbb9cec8b2ccb5031aae94153b8f263916e811524e8a5326f408867a843b7077f624050fabcde93520b26e6f8ceb1dd094aaa0068ed6
+  data.tar.gz: 797b89062dfd212d92ca957b8e51c6dc8acf16ab51e5a60ecb0cbf31dd979c288f2afb71274b2235a24d31da344cfce93b40830d6dc6edd000ce0aa607ce8cde

data/ext/arrow/converters.hpp

@@ -212,7 +212,17 @@ namespace red_arrow {
 
     inline VALUE convert(const arrow::Decimal128Array& array,
                          const int64_t i) {
-      decimal_buffer_ = array.FormatValue(i);
+      return convert_decimal(std::move(array.FormatValue(i)));
+    }
+
+    inline VALUE convert(const arrow::Decimal256Array& array,
+                         const int64_t i) {
+      return convert_decimal(std::move(array.FormatValue(i)));
+    }
+
+  private:
+    inline VALUE convert_decimal(std::string&& value) {
+      decimal_buffer_ = value;
       return rb_funcall(rb_cObject,
                         id_BigDecimal,
                         1,
@@ -221,7 +231,6 @@ namespace red_arrow {
                                        rb_ascii8bit_encoding()));
     }
 
-  private:
     std::string decimal_buffer_;
     ListArrayValueConverter* list_array_value_converter_;
     StructArrayValueConverter* struct_array_value_converter_;
@@ -285,9 +294,11 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
+    VISIT(Decimal256)
     // TODO
     // VISIT(Extension)
 
@@ -339,9 +350,9 @@ namespace red_arrow {
       index_ = index;
       result_ = rb_hash_new();
       const auto struct_type = array.struct_type();
-      const auto n = struct_type->num_children();
+      const auto n = struct_type->num_fields();
       for (int i = 0; i < n; ++i) {
-        const auto field_type = struct_type->child(i).get();
+        const auto field_type = struct_type->field(i).get();
         const auto& field_name = field_type->name();
         auto key_keep = key_;
         key_ = rb_utf8_str_new(field_name.data(), field_name.length());
@@ -388,9 +399,11 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
+    VISIT(Decimal256)
     // TODO
     // VISIT(Extension)
 
@@ -432,10 +445,10 @@ namespace red_arrow {
       index_ = index;
       switch (array.mode()) {
       case arrow::UnionMode::SPARSE:
-        convert_sparse(array);
+        convert_sparse(static_cast<const arrow::SparseUnionArray&>(array));
         break;
       case arrow::UnionMode::DENSE:
-        convert_dense(array);
+        convert_dense(static_cast<const arrow::DenseUnionArray&>(array));
         break;
       default:
         rb_raise(rb_eArgError, "Invalid union mode");
@@ -479,9 +492,11 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
+    VISIT(Decimal256)
     // TODO
     // VISIT(Extension)
 
@@ -501,14 +516,14 @@ namespace red_arrow {
       result_ = result;
     }
 
-    uint8_t compute_child_index(const arrow::UnionArray& array,
+    uint8_t compute_field_index(const arrow::UnionArray& array,
                                 arrow::UnionType* type,
                                 const char* tag) {
       const auto type_code = array.raw_type_codes()[index_];
       if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
-        const auto child_id = type->child_ids()[type_code];
-        if (child_id >= 0) {
-          return child_id;
+        const auto field_id = type->child_ids()[type_code];
+        if (field_id >= 0) {
+          return field_id;
         }
       }
       check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
@@ -516,33 +531,33 @@ namespace red_arrow {
       return 0;
     }
 
-    void convert_sparse(const arrow::UnionArray& array) {
+    void convert_sparse(const arrow::SparseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-sparse-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index).get();
-      check_status(child_array->Accept(this), tag);
+      const auto field_array = array.field(index).get();
+      check_status(field_array->Accept(this), tag);
       field_name_ = field_name_keep;
     }
 
-    void convert_dense(const arrow::UnionArray& array) {
+    void convert_dense(const arrow::DenseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-dense-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index);
+      const auto field_array = array.field(index);
       const auto index_keep = index_;
       index_ = array.value_offset(index_);
-      check_status(child_array->Accept(this), tag);
+      check_status(field_array->Accept(this), tag);
       index_ = index_keep;
       field_name_ = field_name_keep;
     }
@@ -557,30 +572,58 @@ namespace red_arrow {
   public:
     explicit DictionaryArrayValueConverter(ArrayValueConverter* converter)
       : array_value_converter_(converter),
-        index_(0),
+        value_index_(0),
         result_(Qnil) {
     }
 
     VALUE convert(const arrow::DictionaryArray& array,
                   const int64_t index) {
-      index_ = index;
-      auto indices = array.indices().get();
-      check_status(indices->Accept(this),
+      value_index_ = array.GetValueIndex(index);
+      auto dictionary = array.dictionary().get();
+      check_status(dictionary->Accept(this),
                    "[raw-records][dictionary-array]");
       return result_;
     }
 
-    // TODO: Convert to real value.
 #define VISIT(TYPE)                                                     \
     arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
-      result_ = convert_value(array, index_);                           \
+      result_ = convert_value(array, value_index_);                     \
       return arrow::Status::OK();                                       \
       }
 
+    VISIT(Null)
+    VISIT(Boolean)
     VISIT(Int8)
     VISIT(Int16)
     VISIT(Int32)
     VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    // TODO
+    // VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    // TODO
+    // VISIT(Interval)
+    VISIT(List)
+    VISIT(Struct)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    VISIT(Decimal256)
+    // TODO
+    // VISIT(Extension)
 
 #undef VISIT
 
@@ -592,7 +635,7 @@ namespace red_arrow {
     }
 
     ArrayValueConverter* array_value_converter_;
-    int64_t index_;
+    int64_t value_index_;
     VALUE result_;
   };
 

data/ext/arrow/extconf.rb

@@ -16,7 +16,8 @@
 # under the License.
 
 require "extpp"
-require "mkmf-gnome2"
+require "mkmf-gnome"
+require_relative "../../lib/arrow/version"
 
 arrow_pkg_config_path = ENV["ARROW_PKG_CONFIG_PATH"]
 if arrow_pkg_config_path
@@ -24,7 +25,12 @@ if arrow_pkg_config_path
   ENV["PKG_CONFIG_PATH"] = pkg_config_paths.join(File::PATH_SEPARATOR)
 end
 
-unless required_pkg_config_package("arrow",
+unless required_pkg_config_package([
+                                     "arrow",
+                                     Arrow::Version::MAJOR,
+                                     Arrow::Version::MINOR,
+                                     Arrow::Version::MICRO,
+                                   ],
                                    debian: "libarrow-dev",
                                    redhat: "arrow-devel",
                                    homebrew: "apache-arrow",
@@ -32,7 +38,12 @@ unless required_pkg_config_package("arrow",
   exit(false)
 end
 
-unless required_pkg_config_package("arrow-glib",
+unless required_pkg_config_package([
+                                     "arrow-glib",
+                                     Arrow::Version::MAJOR,
+                                     Arrow::Version::MINOR,
+                                     Arrow::Version::MICRO,
+                                   ],
                                    debian: "libarrow-glib-dev",
                                    redhat: "arrow-glib-devel",
                                    homebrew: "apache-arrow-glib",

data/ext/arrow/raw-records.cpp

@@ -100,9 +100,11 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
+      VISIT(Decimal256)
       // TODO
       // VISIT(Extension)
 

data/ext/arrow/values.cpp

@@ -81,9 +81,11 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
+      VISIT(Decimal256)
       // TODO
       // VISIT(Extension)
 

data/lib/arrow/array-builder.rb

@@ -115,6 +115,17 @@ module Arrow
             builder: Date32ArrayBuilder.new,
             detected: true,
           }
+        when BigDecimal
+          if value.to_arrow.is_a?(Decimal128)
+            {
+              builder: Decimal128ArrayBuilder.new,
+            }
+          else
+            {
+              builder: Decimal256ArrayBuilder.new,
+              detected: true,
+            }
+          end
         when ::Array
           sub_builder_info = nil
           value.each do |sub_value|
@@ -194,11 +205,5 @@ module Arrow
         end
       end
     end
-
-    def append_nulls(n)
-      n.times do
-        append_null
-      end
-    end
   end
 end

data/lib/arrow/bigdecimal-extension.rb

@@ -19,6 +19,10 @@ require "bigdecimal"
 
 class BigDecimal
   def to_arrow
-    Arrow::Decimal128.new(to_s)
+    if precision <= Arrow::Decimal128DataType::MAX_PRECISION
+      Arrow::Decimal128.new(to_s)
+    else
+      Arrow::Decimal256.new(to_s)
+    end
   end
 end

data/lib/arrow/buffer.rb

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Buffer
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    def initialize(data)
+      @data = data
+      initialize_raw(data)
+    end
+  end
+end

data/lib/arrow/decimal128-array-builder.rb

@@ -26,36 +26,32 @@ module Arrow
 
     alias_method :append_value_raw, :append_value
     def append_value(value)
-      case value
-      when nil
-        return append_null
-      when String
-        value = Decimal128.new(value)
-      when Float
-        value = Decimal128.new(value.to_s)
-      when BigDecimal
-        value = value.to_arrow
-      end
-      append_value_raw(value)
+      append_value_raw(normalize_value(value))
     end
 
+    alias_method :append_values_raw, :append_values
     def append_values(values, is_valids=nil)
-      if is_valids
-        is_valids.each_with_index do |is_valid, i|
-          if is_valid
-            append_value(values[i])
-          else
-            append_null
-          end
+      if values.is_a?(::Array)
+        values = values.collect do |value|
+          normalize_value(value)
         end
+        append_values_raw(values, is_valids)
       else
-        values.each do |value|
-          if value.nil?
-            append_null
-          else
-            append_value(value)
-          end
-        end
+        append_values_packed(values, is_valids)
+      end
+    end
+
+    private
+    def normalize_value(value)
+      case value
+      when String
+        Decimal128.new(value)
+      when Float
+        Decimal128.new(value.to_s)
+      when BigDecimal
+        Decimal128.new(value.to_s)
+      else
+        value
       end
     end
   end

data/lib/arrow/decimal128-data-type.rb

@@ -17,6 +17,8 @@
 
 module Arrow
   class Decimal128DataType
+    MAX_PRECISION = max_precision
+
     alias_method :initialize_raw, :initialize
     private :initialize_raw
 

data/lib/arrow/decimal128.rb

@@ -38,5 +38,23 @@ module Arrow
         to_s_raw
       end
     end
+
+    alias_method :abs!, :abs
+
+    # @since 3.0.0
+    def abs
+      copied = dup
+      copied.abs!
+      copied
+    end
+
+    alias_method :negate!, :negate
+
+    # @since 3.0.0
+    def negate
+      copied = dup
+      copied.negate!
+      copied
+    end
   end
 end