red-arrow 23.0.1 → 25.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: afda8f4418eddcb71bd239197026cf3553431634c6dbd7a0bf8f7fe8780ba1bb
-  data.tar.gz: 2175cb1036da82770caa753c9e98db5dce72e8a7146d9f248a8a4e971d386462
+  metadata.gz: 3bcc0a7ed27a0a90f4a3675b42017107c61d4d5bdb95967accc6f10369bfc288
+  data.tar.gz: 73c43b33827350528691b2619574e943a6a56a9c643c4407b8bc25285fb2353f
 SHA512:
-  metadata.gz: b29da5a0d440a193ed4c415eb613a5da0e0d8780bcfc6a54b24b17b83f139b56cc16baa90bf6d26f7714f519f1eb56fb7a0038972fc8ecee2c5f9032f51ee13a
-  data.tar.gz: c062071032cc9ab954319284ff624f87eb4a5dd76859f3711ffb5a5db9f42a42e5d9f33aeb0ce7a1c08f27c282f53b67a0e63222f8b571a7379309ce568c28a4
+  metadata.gz: 371fd9f9865dbe7977fbdd2efa2ca0b7dacd22bcb78cc8fd37c2609a5d8843a7335a5ef873caaadfac205ed56fd6eef8ffbee61499a22cd6a3ec16682d82029e
+  data.tar.gz: dc3a6e2e97b9c63af48663dd6d8ee0e8d73348a148c02ae66746dbda93cc2983cd5a97b84c88c784dc7ca513d6e918d3dccf9a13a3a15aabf12c7e8cc4f00acc

data/ext/arrow/arrow.cpp

@@ -63,6 +63,36 @@ namespace red_arrow {
       rbgobj_gc_mark_instance(node->data);
     }
   }
+
+  void
+  call_expression_mark(gpointer object)
+  {
+    auto expression = GARROW_CALL_EXPRESSION(object);
+    auto arguments = garrow_call_expression_get_arguments(expression);
+    for (auto argument = arguments; argument; argument = g_list_next(argument)) {
+      rbgobj_gc_mark_instance(argument->data);
+    }
+  }
+
+  void
+  aggregate_node_options_mark(gpointer object)
+  {
+    auto options = GARROW_AGGREGATE_NODE_OPTIONS(object);
+    auto aggregations = garrow_aggregate_node_options_get_aggregations(options);
+    for (auto aggregation = aggregations; aggregation; aggregation = g_list_next(aggregation)) {
+      rbgobj_gc_mark_instance(aggregation->data);
+    }
+  }
+
+  void
+  project_node_options_mark(gpointer object)
+  {
+    auto options = GARROW_PROJECT_NODE_OPTIONS(object);
+    auto expressions = garrow_project_node_options_get_expressions(options);
+    for (auto expression = expressions; expression; expression = g_list_next(expression)) {
+      rbgobj_gc_mark_instance(expression->data);
+    }
+  }
 }
 
 extern "C" void Init_arrow() {
@@ -124,4 +154,10 @@ extern "C" void Init_arrow() {
                             red_arrow::record_batch_reader_mark);
   rbgobj_register_mark_func(GARROW_TYPE_EXECUTE_PLAN,
                             red_arrow::execute_plan_mark);
+  rbgobj_register_mark_func(GARROW_TYPE_CALL_EXPRESSION,
+                            red_arrow::call_expression_mark);
+  rbgobj_register_mark_func(GARROW_TYPE_AGGREGATE_NODE_OPTIONS,
+                            red_arrow::aggregate_node_options_mark);
+  rbgobj_register_mark_func(GARROW_TYPE_PROJECT_NODE_OPTIONS,
+                            red_arrow::project_node_options_mark);
 }

data/ext/arrow/converters.cpp

@@ -30,6 +30,11 @@ namespace red_arrow {
     return large_list_array_value_converter_->convert(array, i);
   }
 
+  VALUE ArrayValueConverter::convert(const arrow::FixedSizeListArray& array,
+                                     const int64_t i) {
+    return fixed_size_list_array_value_converter_->convert(array, i);
+  }
+
   VALUE ArrayValueConverter::convert(const arrow::StructArray& array,
                                      const int64_t i) {
     return struct_array_value_converter_->convert(array, i);

data/ext/arrow/converters.hpp

@@ -29,6 +29,7 @@
 namespace red_arrow {
   class ListArrayValueConverter;
   class LargeListArrayValueConverter;
+  class FixedSizeListArrayValueConverter;
   class StructArrayValueConverter;
   class MapArrayValueConverter;
   class UnionArrayValueConverter;
@@ -40,6 +41,7 @@ namespace red_arrow {
       : decimal_buffer_(),
         list_array_value_converter_(nullptr),
         large_list_array_value_converter_(nullptr),
+        fixed_size_list_array_value_converter_(nullptr),
         struct_array_value_converter_(nullptr),
         map_array_value_converter_(nullptr),
         union_array_value_converter_(nullptr),
@@ -48,12 +50,14 @@ namespace red_arrow {
 
     inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter,
                                          LargeListArrayValueConverter* large_list_array_value_converter,
+                                         FixedSizeListArrayValueConverter* fixed_size_list_array_value_converter,
                                          StructArrayValueConverter* struct_array_value_converter,
                                          MapArrayValueConverter* map_array_value_converter,
                                          UnionArrayValueConverter* union_array_value_converter,
                                          DictionaryArrayValueConverter* dictionary_array_value_converter) {
       list_array_value_converter_ = list_array_value_converter;
       large_list_array_value_converter_ = large_list_array_value_converter;
+      fixed_size_list_array_value_converter_ = fixed_size_list_array_value_converter;
       struct_array_value_converter_ = struct_array_value_converter;
       map_array_value_converter_ = map_array_value_converter;
       union_array_value_converter_ = union_array_value_converter;
@@ -153,7 +157,15 @@ namespace red_arrow {
                          const int64_t i) {
       int32_t length;
       const auto value = array.GetValue(i, &length);
-      // TODO: encoding support
+      return rb_enc_str_new(reinterpret_cast<const char*>(value),
+                            length,
+                            rb_ascii8bit_encoding());
+    }
+
+    inline VALUE convert(const arrow::LargeBinaryArray& array,
+                         const int64_t i) {
+      int64_t length;
+      const auto value = array.GetValue(i, &length);
       return rb_enc_str_new(reinterpret_cast<const char*>(value),
                             length,
                             rb_ascii8bit_encoding());
@@ -167,6 +179,14 @@ namespace red_arrow {
                              length);
     }
 
+    inline VALUE convert(const arrow::LargeStringArray& array,
+                         const int64_t i) {
+      int64_t length;
+      const auto value = array.GetValue(i, &length);
+      return rb_utf8_str_new(reinterpret_cast<const char*>(value),
+                             length);
+    }
+
     inline VALUE convert(const arrow::FixedSizeBinaryArray& array,
                          const int64_t i) {
       return rb_enc_str_new(reinterpret_cast<const char*>(array.Value(i)),
@@ -225,11 +245,6 @@ namespace red_arrow {
       return rb_time_num_new(sec, Qnil);
     }
 
-    // TODO
-    // inline VALUE convert(const arrow::IntervalArray& array,
-    //                      const int64_t i) {
-    // };
-
     inline VALUE convert(const arrow::MonthIntervalArray& array,
                          const int64_t i) {
       return INT2NUM(array.Value(i));
@@ -264,12 +279,20 @@ namespace red_arrow {
       return value;
     }
 
+    inline VALUE convert(const arrow::DurationArray& array,
+                         const int64_t i) {
+      return LL2NUM(array.Value(i));
+    }
+
     VALUE convert(const arrow::ListArray& array,
                   const int64_t i);
 
     VALUE convert(const arrow::LargeListArray& array,
                   const int64_t i);
 
+    VALUE convert(const arrow::FixedSizeListArray& array,
+                  const int64_t i);
+
     VALUE convert(const arrow::StructArray& array,
                   const int64_t i);
 
@@ -306,6 +329,7 @@ namespace red_arrow {
     std::string decimal_buffer_;
     ListArrayValueConverter* list_array_value_converter_;
     LargeListArrayValueConverter* large_list_array_value_converter_;
+    FixedSizeListArrayValueConverter* fixed_size_list_array_value_converter_;
     StructArrayValueConverter* struct_array_value_converter_;
     MapArrayValueConverter* map_array_value_converter_;
     UnionArrayValueConverter* union_array_value_converter_;
@@ -366,8 +390,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
     VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -465,8 +491,111 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
+    VISIT(List)
+    VISIT(LargeList)
+    VISIT(FixedSizeList)
+    VISIT(Struct)
+    VISIT(Map)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    VISIT(Decimal256)
+    // TODO
+    // VISIT(Extension)
+
+#undef VISIT
+
+  private:
+    template <typename ArrayType>
+    inline VALUE convert_value(const ArrayType& array,
+                               const int64_t i) {
+      return array_value_converter_->convert(array, i);
+    }
+
+    template <typename ArrayType>
+    arrow::Status visit_value(const ArrayType& array) {
+      if (array.null_count() > 0) {
+        for (int64_t i = 0; i < length_; ++i) {
+          auto value = Qnil;
+          if (!array.IsNull(i + offset_)) {
+            value = convert_value(array, i + offset_);
+          }
+          rb_ary_push(result_, value);
+        }
+      } else {
+        for (int64_t i = 0; i < length_; ++i) {
+          rb_ary_push(result_, convert_value(array, i + offset_));
+        }
+      }
+      return arrow::Status::OK();
+    }
+
+    ArrayValueConverter* array_value_converter_;
+    int32_t offset_;
+    int32_t length_;
+    VALUE result_;
+  };
+
+  class FixedSizeListArrayValueConverter : public arrow::ArrayVisitor {
+  public:
+    explicit FixedSizeListArrayValueConverter(ArrayValueConverter* converter)
+      : array_value_converter_(converter),
+        offset_(0),
+        length_(0),
+        result_(Qnil) {}
+
+    VALUE convert(const arrow::FixedSizeListArray& array, const int64_t index) {
+      auto values = array.values().get();
+      auto offset_keep = offset_;
+      auto length_keep = length_;
+      offset_ = array.value_offset(index);
+      length_ = array.value_length(index);
+      auto result_keep = result_;
+      result_ = rb_ary_new_capa(length_);
+      check_status(values->Accept(this),
+                   "[raw-records][fixed-size-list-array]");
+      offset_ = offset_keep;
+      length_ = length_keep;
+      auto result_return = result_;
+      result_ = result_keep;
+      return result_return;
+    }
+
+#define VISIT(TYPE)                                                     \
+    arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
+      return visit_value(array);                                        \
+    }
+
+    VISIT(Null)
+    VISIT(Boolean)
+    VISIT(Int8)
+    VISIT(Int16)
+    VISIT(Int32)
+    VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
     VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -572,8 +701,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
     VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -675,8 +806,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
     VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -779,8 +912,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
     VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -881,7 +1016,9 @@ namespace red_arrow {
     VISIT(Float)
     VISIT(Double)
     VISIT(Binary)
+    VISIT(LargeBinary)
     VISIT(String)
+    VISIT(LargeString)
     VISIT(FixedSizeBinary)
     VISIT(Date32)
     VISIT(Date64)
@@ -891,8 +1028,10 @@ namespace red_arrow {
     VISIT(MonthInterval)
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
+    VISIT(Duration)
     VISIT(List)
     VISIT(LargeList)
+    VISIT(FixedSizeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -923,6 +1062,7 @@ namespace red_arrow {
       : array_value_converter_(),
         list_array_value_converter_(&array_value_converter_),
         large_list_array_value_converter_(&array_value_converter_),
+        fixed_size_list_array_value_converter_(&array_value_converter_),
         struct_array_value_converter_(&array_value_converter_),
         map_array_value_converter_(&array_value_converter_),
         union_array_value_converter_(&array_value_converter_),
@@ -930,6 +1070,7 @@ namespace red_arrow {
       array_value_converter_.
         set_sub_value_converters(&list_array_value_converter_,
                                  &large_list_array_value_converter_,
+                                 &fixed_size_list_array_value_converter_,
                                  &struct_array_value_converter_,
                                  &map_array_value_converter_,
                                  &union_array_value_converter_,
@@ -945,6 +1086,7 @@ namespace red_arrow {
     ArrayValueConverter array_value_converter_;
     ListArrayValueConverter list_array_value_converter_;
     LargeListArrayValueConverter large_list_array_value_converter_;
+    FixedSizeListArrayValueConverter fixed_size_list_array_value_converter_;
     StructArrayValueConverter struct_array_value_converter_;
     MapArrayValueConverter map_array_value_converter_;
     UnionArrayValueConverter union_array_value_converter_;

data/ext/arrow/memory-view.cpp

@@ -31,6 +31,7 @@
 #  undef private
 #endif
 
+#include <cstring>
 #include <sstream>
 
 namespace red_arrow {
@@ -220,6 +221,7 @@ namespace red_arrow {
         return false;
       }
       auto view_ = reinterpret_cast<memory_view *>(view);
+      memset(view_, 0, sizeof(memory_view));
       view_->obj = obj;
       view_->private_data = new PrivateData();
       auto array = GARROW_ARRAY(RVAL2GOBJ(obj));
@@ -231,9 +233,6 @@ namespace red_arrow {
       }
       view_->readonly = true;
       view_->ndim = 1;
-      view_->shape = NULL;
-      view_->strides = NULL;
-      view_->sub_offsets = NULL;
       return true;
     }
 
@@ -258,6 +257,7 @@ namespace red_arrow {
         return false;
       }
       auto view_ = reinterpret_cast<memory_view *>(view);
+      memset(view_, 0, sizeof(memory_view));
       view_->obj = obj;
       auto buffer = GARROW_BUFFER(RVAL2GOBJ(obj));
       auto arrow_buffer = garrow_buffer_get_raw(buffer);
@@ -275,9 +275,6 @@ namespace red_arrow {
       view_->byte_size = arrow_buffer->size();
       view_->readonly = true;
       view_->ndim = 1;
-      view_->shape = NULL;
-      view_->strides = NULL;
-      view_->sub_offsets = NULL;
       return true;
     }
 

data/ext/arrow/raw-records.cpp

@@ -88,7 +88,9 @@ namespace red_arrow {
       VISIT(Float)
       VISIT(Double)
       VISIT(Binary)
+      VISIT(LargeBinary)
       VISIT(String)
+      VISIT(LargeString)
       VISIT(FixedSizeBinary)
       VISIT(Date32)
       VISIT(Date64)
@@ -98,7 +100,10 @@ namespace red_arrow {
       VISIT(MonthInterval)
       VISIT(DayTimeInterval)
       VISIT(MonthDayNanoInterval)
+      VISIT(Duration)
       VISIT(List)
+      VISIT(LargeList)
+      VISIT(FixedSizeList)
       VISIT(Struct)
       VISIT(Map)
       VISIT(SparseUnion)
@@ -224,7 +229,9 @@ namespace red_arrow {
       VISIT(Float)
       VISIT(Double)
       VISIT(Binary)
+      VISIT(LargeBinary)
       VISIT(String)
+      VISIT(LargeString)
       VISIT(FixedSizeBinary)
       VISIT(Date32)
       VISIT(Date64)
@@ -234,7 +241,10 @@ namespace red_arrow {
       VISIT(MonthInterval)
       VISIT(DayTimeInterval)
       VISIT(MonthDayNanoInterval)
+      VISIT(Duration)
       VISIT(List)
+      VISIT(LargeList)
+      VISIT(FixedSizeList)
       VISIT(Struct)
       VISIT(Map)
       VISIT(SparseUnion)

data/ext/arrow/values.cpp

@@ -69,7 +69,9 @@ namespace red_arrow {
       VISIT(Float)
       VISIT(Double)
       VISIT(Binary)
+      VISIT(LargeBinary)
       VISIT(String)
+      VISIT(LargeString)
       VISIT(FixedSizeBinary)
       VISIT(Date32)
       VISIT(Date64)
@@ -79,8 +81,10 @@ namespace red_arrow {
       VISIT(MonthInterval)
       VISIT(DayTimeInterval)
       VISIT(MonthDayNanoInterval)
+      VISIT(Duration)
       VISIT(List)
       VISIT(LargeList)
+      VISIT(FixedSizeList)
       VISIT(Struct)
       VISIT(Map)
       VISIT(SparseUnion)

data/lib/arrow/column-containable.rb

@@ -152,5 +152,70 @@ module Arrow
     def column_names
       @column_names ||= columns.collect(&:name)
     end
+
+    # Merges columns from the given container or Hash and creates
+    # a new container.
+    #
+    # @param other [Hash, self]
+    #   The columns to be merged.
+    #
+    # @return [self]
+    def merge(other)
+      added_columns = {}
+      removed_columns = {}
+
+      case other
+      when Hash
+        other.each do |name, value|
+          name = name.to_s
+          if value
+            added_columns[name] = ensure_raw_column(name, value)
+          else
+            removed_columns[name] = true
+          end
+        end
+      when self.class
+        other.columns.each do |column|
+          name = column.name
+          added_columns[name] = ensure_raw_column(name, column)
+        end
+      else
+        message = "merge target must be Hash or #{self.class}: " +
+          "<#{other.inspect}>: #{inspect}"
+        raise ArgumentError, message
+      end
+
+      new_columns = []
+
+      columns.each do |column|
+        column_name = column.name
+        new_column = added_columns.delete(column_name)
+
+        if new_column
+          new_columns << new_column
+          next
+        end
+
+        next if removed_columns.key?(column_name)
+
+        new_columns << ensure_raw_column(column_name, column)
+      end
+
+      added_columns.each_value do |new_column|
+        new_columns << new_column
+      end
+
+      new_fields = []
+      new_arrays = []
+
+      new_columns.each do |new_column|
+        new_fields << new_column[:field]
+        new_arrays << new_column[:data]
+      end
+
+      merged = self.class.new(new_fields, new_arrays)
+      share_input(merged)
+      merged
+    end
   end
 end

data/lib/arrow/column.rb

@@ -42,6 +42,10 @@ module Arrow
       @data.to_arrow_chunked_array
     end
 
+    def values
+      @data.values
+    end
+
     def name
       @field.name
     end

data/lib/arrow/dense-union-array.rb

@@ -19,7 +19,7 @@ module Arrow
   class DenseUnionArray
     def get_value(i)
       child_id = get_child_id(i)
-      field = get_field(child_id)
+      field = fields[child_id]
       field[get_value_offset(i)]
     end
   end

data/lib/arrow/fixed-size-list-data-type.rb

@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class FixedSizeListDataType
+    include ListFieldResolvable
+
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    # Creates a new {Arrow::FixedSizeListDataType}.
+    #
+    # @overload initialize(field, size)
+    #
+    #   @param field [Arrow::Field, Hash] The field of the fixed size
+    #     list data type. You can also specify field description by
+    #     `Hash`.
+    #
+    #     See {Arrow::Field.new} how to specify field description.
+    #
+    #   @param size [Integer] The number of values in each element.
+    #
+    #   @example Create a fixed size list data type with {Arrow::Field}
+    #     visible_field = Arrow::Field.new("visible", :boolean)
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new(visible_field, size)
+    #
+    #   @example Create a list data type with field description
+    #     description = {name: "visible", type: :boolean}
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new(description, size)
+    #
+    # @overload initialize(description)
+    #
+    #   @param description [Hash] The description of the fixed size
+    #     list data type. It must have `:field` value and `:size`
+    #     value.
+    #
+    #   @option description [Arrow::Field, Hash] :field The field of
+    #     the list data type. You can also specify field description
+    #     by `Hash`.
+    #
+    #     See {Arrow::Field.new} how to specify field description.
+    #
+    #   @option description [Integer] :size The number of values of
+    #     each element of the fixed size list data type.
+    #
+    #   @example Create a fixed size list data type with {Arrow::Field}
+    #     visible_field = Arrow::Field.new("visible", :boolean)
+    #     Arrow::FixedSizeListDataType.new(field: visible_field, size: 2)
+    #
+    #   @example Create a fixed size list data type with field description
+    #     Arrow::FixedSizeListDataType.new(field: {
+    #                                        name: "visible",
+    #                                        type: :boolean,
+    #                                      },
+    #                                      size: 2)
+    #
+    # @overload initialize(data_type, size)
+    #
+    #   @param data_type [Arrow::DataType, String, Symbol,
+    #     ::Array<String>, ::Array<Symbol>, Hash] The element data
+    #     type of the fixed size list data type. A field is created
+    #     with the default name `"item"` from the data type
+    #     automatically.
+    #
+    #     See {Arrow::DataType.resolve} how to specify data type.
+    #
+    #   @param size [Integer] The number of values in each
+    #     element.
+    #
+    #   @example Create a fixed size list data type with {Arrow::DataType}
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new(Arrow::BooleanDataType.new,
+    #                                      size)
+    #
+    #   @example Create a fixed size list data type with data type name as String
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new("boolean", size)
+    #
+    #   @example Create a fixed size list data type with data type name as Symbol
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new(:boolean, size)
+    #
+    #   @example Create a fixed size list data type with data type as Array
+    #     size = 2
+    #     Arrow::FixedSizeListDataType.new([:time32, :milli], size)
+    def initialize(*args)
+      n_args = args.size
+      case n_args
+      when 1
+        description = args[0]
+        size = description.delete(:size)
+        initialize_raw(resolve_field(description), size)
+      when 2
+        field, size = args
+        initialize_raw(resolve_field(field), size)
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 1..2)"
+        raise ArgumentError, message
+      end
+    end
+  end
+end

data/lib/arrow/libraries.rb

@@ -71,6 +71,7 @@ require_relative "file-system"
 require_relative "fixed-size-binary-array"
 require_relative "fixed-size-binary-array-builder"
 require_relative "fixed-size-list-array-builder"
+require_relative "fixed-size-list-data-type"
 require_relative "function"
 require_relative "group"
 require_relative "half-float"
@@ -134,5 +135,6 @@ require_relative "timestamp-array"
 require_relative "timestamp-array-builder"
 require_relative "timestamp-data-type"
 require_relative "timestamp-parser"
+require_relative "union-array"
 require_relative "union-array-builder"
 require_relative "writable"

data/lib/arrow/record-batch.rb

@@ -37,7 +37,12 @@ module Arrow
           super(schema, n_rows, values)
         when 2
           schema, data = args
-          RecordBatchBuilder.build(schema, data)
+          schema = Schema.new(schema) unless schema.is_a?(Schema)
+          if !data.empty? and data.all? {|array| array.is_a?(Arrow::Array)}
+            super(schema, data[0].size, data)
+          else
+            RecordBatchBuilder.build(schema, data)
+          end
         when 3
           super
         else
@@ -75,5 +80,26 @@ module Arrow
       end
       super
     end
+
+    private
+
+    def ensure_raw_column(name, data)
+      case data
+      when Array
+        {
+          field: Field.new(name, data.value_data_type),
+          data: data,
+        }
+      when Column
+        {
+          field: data.field,
+          data: data.data,
+        }
+      else
+        message = "column must be Arrow::Array or Arrow::Column: " +
+          "<#{name}>: <#{data.inspect}>: #{inspect}"
+        raise ArgumentError, message
+      end
+    end
   end
 end

data/lib/arrow/sort-key.rb

@@ -46,16 +46,16 @@ module Arrow
       #   @return [Arrow::SortKey] A new suitable sort key.
       #
       # @since 4.0.0
-      def resolve(target, order=nil)
+      def resolve(target, order=nil, null_placement=nil)
         return target if target.is_a?(self)
-        new(target, order)
+        new(target, order, null_placement)
       end
 
       # @api private
       def try_convert(value)
         case value
         when Symbol, String
-          new(value.to_s, :ascending)
+          new(value.to_s, :ascending, :at_end)
         else
           nil
         end
@@ -71,37 +71,46 @@ module Arrow
     #   @param target [Symbol, String] The name or dot path of the
     #     sort column.
     #
-    #     If `target` is a String, the first character may be
-    #     processed as the "leading order mark". If the first
-    #     character is `"+"` or `"-"`, they are processed as a leading
-    #     order mark. If the first character is processed as a leading
-    #     order mark, the first character is removed from sort column
-    #     target and corresponding order is used. `"+"` uses ascending
-    #     order and `"-"` uses ascending order.
+    #     If `target` is a String, it may have prefix markers that specify
+    #     the sort order and null placement. The format is `[+/-][^/$]column`:
     #
-    #     If `target` is either not a String or `target` doesn't start
-    #     with the leading order mark, sort column is `target` as-is
-    #     and ascending order is used.
+    #     - `"+"` prefix means ascending order
+    #     - `"-"` prefix means descending order
+    #     - `"^"` prefix means nulls at start
+    #     - `"$"` prefix means nulls at end
     #
-    #   @example String without the leading order mark
-    #     key = Arrow::SortKey.new("count")
-    #     key.target # => "count"
-    #     key.order  # => Arrow::SortOrder::ASCENDING
+    #     If `target` is a Symbol, it is converted to String and used as-is
+    #     (no prefix processing).
     #
-    #   @example String with the "+" leading order mark
-    #     key = Arrow::SortKey.new("+count")
-    #     key.target # => "count"
-    #     key.order  # => Arrow::SortOrder::ASCENDING
+    #   @example String without any prefix
+    #     key = Arrow::SortKey.new("count")
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::ASCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
-    #   @example String with the "-" leading order mark
+    #   @example String with order prefix only
     #     key = Arrow::SortKey.new("-count")
-    #     key.target # => "count"
-    #     key.order  # => Arrow::SortOrder::DESCENDING
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
+    #
+    #   @example String with order and null placement prefixes
+    #     key = Arrow::SortKey.new("-^count")
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_START
     #
-    #   @example Symbol that starts with "-"
+    #   @example String with null placement prefix only
+    #     key = Arrow::SortKey.new("^count")
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::ASCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_START
+    #
+    #   @example Symbol (no prefix processing)
     #     key = Arrow::SortKey.new(:"-count")
-    #     key.target # => "-count"
-    #     key.order  # => Arrow::SortOrder::ASCENDING
+    #     key.target         # => "-count"
+    #     key.order          # => Arrow::SortOrder::ASCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
     # @overload initialize(target, order)
     #
@@ -122,27 +131,54 @@ module Arrow
     #     key = Arrow::SortKey.new("-count", :ascending)
     #     key.target # => "-count"
     #     key.order  # => Arrow::SortOrder::ASCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
     #   @example Order by abbreviated target with Symbol
     #     key = Arrow::SortKey.new("count", :desc)
     #     key.target # => "count"
     #     key.order  # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
     #   @example Order by String
     #     key = Arrow::SortKey.new("count", "descending")
     #     key.target # => "count"
     #     key.order  # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
-    #   @example Order by Arrow::SortOrder
-    #     key = Arrow::SortKey.new("count", Arrow::SortOrder::DESCENDING)
+    #   @example Order by Arrow::SortOrder, give null_placement with target
+    #     key = Arrow::SortKey.new("^count", Arrow::SortOrder::DESCENDING)
     #     key.target # => "count"
     #     key.order  # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_START
+    #
+    # @overload initialize(target, order, null_placement)
+    #
+    #   @param target [Symbol, String] The name or dot path of the
+    #     sort column.
+    #
+    #   @param order [Symbol, String, Arrow::SortOrder] How to order
+    #     by this sort key.
+    #
+    #     If this is a Symbol or String, this must be `:ascending`,
+    #     `"ascending"`, `:asc`, `"asc"`, `:descending`,
+    #     `"descending"`, `:desc` or `"desc"`.
+    #
+    #   @param null_placement [Symbol, String, Arrow::NullPlacement]
+    #     Where to place nulls and NaNs. Must be `:at_start`, `"at_start"`,
+    #     `:at_end`, or `"at_end"`.
+    #
+    #   @example With all explicit parameters
+    #     key = Arrow::SortKey.new("count", :desc, :at_start)
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_START
     #
     # @since 4.0.0
-    def initialize(target, order=nil)
-      target, order = normalize_target(target, order)
+    def initialize(target, order=nil, null_placement=nil)
+      target, order, null_placement = normalize_target(target, order, null_placement)
       order = normalize_order(order) || :ascending
-      initialize_raw(target, order)
+      null_placement = normalize_null_placement(null_placement) || :at_end
+      initialize_raw(target, order, null_placement)
     end
 
     # @return [String] The string representation of this sort key. You
@@ -151,37 +187,66 @@ module Arrow
     #
     # @example Recreate Arrow::SortKey
     #   key = Arrow::SortKey.new("-count")
-    #   key.to_s # => "-count"
+    #   key.to_s # => "-$count"
     #   key == Arrow::SortKey.new(key.to_s) # => true
     #
     # @since 4.0.0
     def to_s
+      result = ""
       if order == SortOrder::ASCENDING
-        "+#{target}"
+        result += "+"
+      else
+        result += "-"
+      end
+      if null_placement == NullPlacement::AT_START
+        result += "^"
       else
-        "-#{target}"
+        result += "$"
       end
+      result += target
+      result
     end
 
     # For backward compatibility
     alias_method :name, :target
 
     private
-    def normalize_target(target, order)
+    # Parse prefix format: [+/-][^/$]column
+    # Examples: -$column, +^column, ^column, -column
+    #
+    # Only strips prefixes if the corresponding parameter is not already set.
+    # This preserves backward compatibility where specifying order explicitly
+    # means the target is used as-is for order prefixes.
+    def normalize_target(target, order, null_placement)
       case target
       when Symbol
-        return target.to_s, order
+        return target.to_s, order, null_placement
       when String
-        return target, order if order
-        if target.start_with?("-")
-          return target[1..-1], order || :descending
-        elsif target.start_with?("+")
-          return target[1..-1], order || :ascending
-        else
-          return target, order
+        remaining = target
+
+        unless order
+          if remaining.start_with?("-")
+            order = :descending
+            remaining = remaining[1..-1]
+          elsif remaining.start_with?("+")
+            order = :ascending
+            remaining = remaining[1..-1]
+          end
+        end
+
+        unless null_placement
+          if remaining.start_with?("^")
+            null_placement = :at_start
+            remaining = remaining[1..-1]
+          elsif remaining.start_with?("$")
+            null_placement = :at_end
+            remaining = remaining[1..-1]
+          end
         end
+
+        return remaining, order, null_placement
       else
-        return target, order
+        return target, order, null_placement
       end
     end
 
@@ -195,5 +260,16 @@ module Arrow
         order
       end
     end
+    
+    def normalize_null_placement(null_placement)
+      case null_placement
+      when :at_end, "at_end"
+        :at_end
+      when :at_start, "at_start"
+        :at_start
+      else
+        null_placement
+      end
+    end
   end
 end

data/lib/arrow/sort-options.rb

@@ -102,8 +102,8 @@ module Arrow
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
     # @since 4.0.0
-    def add_sort_key(target, order=nil)
-      add_sort_key_raw(SortKey.resolve(target, order))
+    def add_sort_key(target, order=nil, null_placement=nil)
+      add_sort_key_raw(SortKey.resolve(target, order, null_placement))
     end
   end
 end

data/lib/arrow/sparse-union-array.rb

@@ -19,7 +19,7 @@ module Arrow
   class SparseUnionArray
     def get_value(i)
       child_id = get_child_id(i)
-      field = get_field(child_id)
+      field = fields[child_id]
       field[i]
     end
   end

data/lib/arrow/table-formatter.rb

@@ -80,7 +80,11 @@ module Arrow
         when nil
           "%*s" % [width, FORMATTED_NULL]
         else
-          "%-*s" % [width, value.to_s]
+          value = value.to_s
+          if value.encoding == Encoding::ASCII_8BIT
+            value = value.each_byte.collect {|byte| "%X" % byte}.join
+          end
+          "%-*s" % [width, value]
         end
       end
 

data/lib/arrow/table-saver.rb

@@ -130,9 +130,9 @@ module Arrow
       end
     end
 
-    def save_raw(writer_class)
+    def save_raw(writer_class, *args)
       open_output_stream do |output|
-        writer_class.open(output, @table.schema) do |writer|
+        writer_class.open(output, @table.schema, *args) do |writer|
           writer.write_table(@table)
         end
       end
@@ -144,7 +144,7 @@ module Arrow
 
     # @since 1.0.0
     def save_as_arrow_file
-      save_raw(RecordBatchFileWriter)
+      save_raw(RecordBatchFileWriter, nil, @options[:metadata])
     end
 
     # @deprecated Use `format: :arrow_batch` instead.

data/lib/arrow/table.rb

@@ -354,60 +354,6 @@ module Arrow
       sliced_table
     end
 
-    # TODO
-    #
-    # @return [Arrow::Table]
-    def merge(other)
-      added_columns = {}
-      removed_columns = {}
-
-      case other
-      when Hash
-        other.each do |name, value|
-          name = name.to_s
-          if value
-            added_columns[name] = ensure_raw_column(name, value)
-          else
-            removed_columns[name] = true
-          end
-        end
-      when Table
-        added_columns = {}
-        other.columns.each do |column|
-          name = column.name
-          added_columns[name] = ensure_raw_column(name, column)
-        end
-      else
-        message = "merge target must be Hash or Arrow::Table: " +
-          "<#{other.inspect}>: #{inspect}"
-        raise ArgumentError, message
-      end
-
-      new_columns = []
-      columns.each do |column|
-        column_name = column.name
-        new_column = added_columns.delete(column_name)
-        if new_column
-          new_columns << new_column
-          next
-        end
-        next if removed_columns.key?(column_name)
-        new_columns << ensure_raw_column(column_name, column)
-      end
-      added_columns.each do |name, new_column|
-        new_columns << new_column
-      end
-      new_fields = []
-      new_arrays = []
-      new_columns.each do |new_column|
-        new_fields << new_column[:field]
-        new_arrays << new_column[:data]
-      end
-      table = self.class.new(new_fields, new_arrays)
-      share_input(table)
-      table
-    end
-
     alias_method :remove_column_raw, :remove_column
     def remove_column(name_or_index)
       case name_or_index

data/lib/arrow/union-array.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class UnionArray
+    def fields
+      @fields ||= n_fields.times.collect do |i|
+        get_field(i)
+      end
+    end
+  end
+end

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "23.0.1"
+  VERSION = "25.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -58,7 +58,7 @@ Gem::Specification.new do |spec|
     spec.requirements << "jar org.apache.arrow, arrow-vector, #{spec.version}"
     spec.requirements << "jar org.apache.arrow, arrow-memory-netty, #{spec.version}"
   else
-    spec.add_runtime_dependency("extpp", ">= 0.1.1")
+    spec.add_runtime_dependency("extpp", ">= 0.1.2")
     spec.add_runtime_dependency("gio2", ">= 4.2.3")
     spec.add_runtime_dependency("pkg-config")
 
@@ -98,6 +98,8 @@ Gem::Specification.new do |spec|
 
       ["fedora", "libarrow-glib-devel"],
 
+      ["homebrew", "apache-arrow-glib"],
+
       # Try without additional repository
       ["rhel", "arrow-glib-devel"],
       # Retry with additional repository

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 23.0.1
+  version: 25.0.0
 platform: ruby
 authors:
 - The Apache Software Foundation
@@ -43,14 +43,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: gio2
   requirement: !ruby/object:Gem::Requirement
@@ -155,6 +155,7 @@ files:
 - lib/arrow/fixed-size-binary-array-builder.rb
 - lib/arrow/fixed-size-binary-array.rb
 - lib/arrow/fixed-size-list-array-builder.rb
+- lib/arrow/fixed-size-list-data-type.rb
 - lib/arrow/function.rb
 - lib/arrow/generic-filterable.rb
 - lib/arrow/generic-takeable.rb
@@ -250,6 +251,7 @@ files:
 - lib/arrow/timestamp-data-type.rb
 - lib/arrow/timestamp-parser.rb
 - lib/arrow/union-array-builder.rb
+- lib/arrow/union-array.rb
 - lib/arrow/version.rb
 - lib/arrow/writable.rb
 - red-arrow.gemspec
@@ -257,7 +259,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata:
-  msys2_mingw_dependencies: arrow>=23.0.1
+  msys2_mingw_dependencies: arrow>=25.0.0
 rdoc_options: []
 require_paths:
 - lib
@@ -272,20 +274,21 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- 'system: arrow-glib>=23.0.1: amazon_linux: arrow-glib-devel'
-- 'system: arrow-glib>=23.0.1: amazon_linux: https://packages.apache.org/artifactory/arrow/amazon-linux/%{version}/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=23.0.1: amazon_linux: arrow-glib-devel'
-- 'system: arrow-glib>=23.0.1: centos: arrow-glib-devel'
-- 'system: arrow-glib>=23.0.1: centos: https://packages.apache.org/artifactory/arrow/centos/%{major_version}-stream/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=23.0.1: centos: arrow-glib-devel'
-- 'system: arrow-glib>=23.0.1: conda: arrow-c-glib'
-- 'system: arrow-glib>=23.0.1: debian: libarrow-glib-dev'
-- 'system: arrow-glib>=23.0.1: debian: https://packages.apache.org/artifactory/arrow/%{distribution}/apache-arrow-apt-source-latest-%{code_name}.deb'
-- 'system: arrow-glib>=23.0.1: debian: libarrow-glib-dev'
-- 'system: arrow-glib>=23.0.1: fedora: libarrow-glib-devel'
-- 'system: arrow-glib>=23.0.1: rhel: arrow-glib-devel'
-- 'system: arrow-glib>=23.0.1: rhel: https://packages.apache.org/artifactory/arrow/almalinux/%{major_version}/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=23.0.1: rhel: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: amazon_linux: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: amazon_linux: https://packages.apache.org/artifactory/arrow/amazon-linux/%{version}/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=25.0.0: amazon_linux: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: centos: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: centos: https://packages.apache.org/artifactory/arrow/centos/%{major_version}-stream/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=25.0.0: centos: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: conda: arrow-c-glib'
+- 'system: arrow-glib>=25.0.0: debian: libarrow-glib-dev'
+- 'system: arrow-glib>=25.0.0: debian: https://packages.apache.org/artifactory/arrow/%{distribution}/apache-arrow-apt-source-latest-%{code_name}.deb'
+- 'system: arrow-glib>=25.0.0: debian: libarrow-glib-dev'
+- 'system: arrow-glib>=25.0.0: fedora: libarrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: homebrew: apache-arrow-glib'
+- 'system: arrow-glib>=25.0.0: rhel: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: rhel: https://packages.apache.org/artifactory/arrow/almalinux/%{major_version}/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=25.0.0: rhel: arrow-glib-devel'
 rubygems_version: 3.6.7
 specification_version: 4
 summary: Red Arrow is the Ruby bindings of Apache Arrow