red-arrow 24.0.0 → 25.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8464767793fb8efd382689c8ca939af17fdb052f462885efb0461d07d905d18
-  data.tar.gz: e6d037e1400af61e90438eaa255c172eaf850b136ea7f8262b6acbd2b41c8126
+  metadata.gz: 3bcc0a7ed27a0a90f4a3675b42017107c61d4d5bdb95967accc6f10369bfc288
+  data.tar.gz: 73c43b33827350528691b2619574e943a6a56a9c643c4407b8bc25285fb2353f
 SHA512:
-  metadata.gz: 0cfab03a5b046c67d892db13cf1cd8d988fd0da9e61e950983dc2c1662c9b25a11ad8fb2e27ca68c7cd34814f208a4a51eda0d689e6aa9c514b82c026797ac7e
-  data.tar.gz: 22370cbfb633c8efd7d3ef8e8131001ece304d1c2e79e29bbda7eae4095dea0486f79a9253fbf8d586c11389bd2f3c8b38841f90733cb1dfb4787c2a9327fa15
+  metadata.gz: 371fd9f9865dbe7977fbdd2efa2ca0b7dacd22bcb78cc8fd37c2609a5d8843a7335a5ef873caaadfac205ed56fd6eef8ffbee61499a22cd6a3ec16682d82029e
+  data.tar.gz: dc3a6e2e97b9c63af48663dd6d8ee0e8d73348a148c02ae66746dbda93cc2983cd5a97b84c88c784dc7ca513d6e918d3dccf9a13a3a15aabf12c7e8cc4f00acc

data/ext/arrow/memory-view.cpp

@@ -31,6 +31,7 @@
 #  undef private
 #endif
 
+#include <cstring>
 #include <sstream>
 
 namespace red_arrow {
@@ -220,6 +221,7 @@ namespace red_arrow {
         return false;
       }
       auto view_ = reinterpret_cast<memory_view *>(view);
+      memset(view_, 0, sizeof(memory_view));
       view_->obj = obj;
       view_->private_data = new PrivateData();
       auto array = GARROW_ARRAY(RVAL2GOBJ(obj));
@@ -231,9 +233,6 @@ namespace red_arrow {
       }
       view_->readonly = true;
       view_->ndim = 1;
-      view_->shape = NULL;
-      view_->strides = NULL;
-      view_->sub_offsets = NULL;
       return true;
     }
 
@@ -258,6 +257,7 @@ namespace red_arrow {
         return false;
       }
       auto view_ = reinterpret_cast<memory_view *>(view);
+      memset(view_, 0, sizeof(memory_view));
       view_->obj = obj;
       auto buffer = GARROW_BUFFER(RVAL2GOBJ(obj));
       auto arrow_buffer = garrow_buffer_get_raw(buffer);
@@ -275,9 +275,6 @@ namespace red_arrow {
       view_->byte_size = arrow_buffer->size();
       view_->readonly = true;
       view_->ndim = 1;
-      view_->shape = NULL;
-      view_->strides = NULL;
-      view_->sub_offsets = NULL;
       return true;
     }
 

data/lib/arrow/column-containable.rb

@@ -152,5 +152,70 @@ module Arrow
     def column_names
       @column_names ||= columns.collect(&:name)
     end
+
+    # Merges columns from the given container or Hash and creates
+    # a new container.
+    #
+    # @param other [Hash, self]
+    #   The columns to be merged.
+    #
+    # @return [self]
+    def merge(other)
+      added_columns = {}
+      removed_columns = {}
+
+      case other
+      when Hash
+        other.each do |name, value|
+          name = name.to_s
+          if value
+            added_columns[name] = ensure_raw_column(name, value)
+          else
+            removed_columns[name] = true
+          end
+        end
+      when self.class
+        other.columns.each do |column|
+          name = column.name
+          added_columns[name] = ensure_raw_column(name, column)
+        end
+      else
+        message = "merge target must be Hash or #{self.class}: " +
+          "<#{other.inspect}>: #{inspect}"
+        raise ArgumentError, message
+      end
+
+      new_columns = []
+
+      columns.each do |column|
+        column_name = column.name
+        new_column = added_columns.delete(column_name)
+
+        if new_column
+          new_columns << new_column
+          next
+        end
+
+        next if removed_columns.key?(column_name)
+
+        new_columns << ensure_raw_column(column_name, column)
+      end
+
+      added_columns.each_value do |new_column|
+        new_columns << new_column
+      end
+
+      new_fields = []
+      new_arrays = []
+
+      new_columns.each do |new_column|
+        new_fields << new_column[:field]
+        new_arrays << new_column[:data]
+      end
+
+      merged = self.class.new(new_fields, new_arrays)
+      share_input(merged)
+      merged
+    end
   end
 end

data/lib/arrow/record-batch.rb

@@ -37,7 +37,12 @@ module Arrow
           super(schema, n_rows, values)
         when 2
           schema, data = args
-          RecordBatchBuilder.build(schema, data)
+          schema = Schema.new(schema) unless schema.is_a?(Schema)
+          if !data.empty? and data.all? {|array| array.is_a?(Arrow::Array)}
+            super(schema, data[0].size, data)
+          else
+            RecordBatchBuilder.build(schema, data)
+          end
         when 3
           super
         else
@@ -75,5 +80,26 @@ module Arrow
       end
       super
     end
+
+    private
+
+    def ensure_raw_column(name, data)
+      case data
+      when Array
+        {
+          field: Field.new(name, data.value_data_type),
+          data: data,
+        }
+      when Column
+        {
+          field: data.field,
+          data: data.data,
+        }
+      else
+        message = "column must be Arrow::Array or Arrow::Column: " +
+          "<#{name}>: <#{data.inspect}>: #{inspect}"
+        raise ArgumentError, message
+      end
+    end
   end
 end

data/lib/arrow/sort-key.rb

@@ -46,16 +46,16 @@ module Arrow
       #   @return [Arrow::SortKey] A new suitable sort key.
       #
       # @since 4.0.0
-      def resolve(target, order=nil)
+      def resolve(target, order=nil, null_placement=nil)
         return target if target.is_a?(self)
-        new(target, order)
+        new(target, order, null_placement)
       end
 
       # @api private
       def try_convert(value)
         case value
         when Symbol, String
-          new(value.to_s, :ascending)
+          new(value.to_s, :ascending, :at_end)
         else
           nil
         end
@@ -71,37 +71,46 @@ module Arrow
     #   @param target [Symbol, String] The name or dot path of the
     #     sort column.
     #
-    #     If `target` is a String, the first character may be
-    #     processed as the "leading order mark". If the first
-    #     character is `"+"` or `"-"`, they are processed as a leading
-    #     order mark. If the first character is processed as a leading
-    #     order mark, the first character is removed from sort column
-    #     target and corresponding order is used. `"+"` uses ascending
-    #     order and `"-"` uses ascending order.
+    #     If `target` is a String, it may have prefix markers that specify
+    #     the sort order and null placement. The format is `[+/-][^/$]column`:
     #
-    #     If `target` is either not a String or `target` doesn't start
-    #     with the leading order mark, sort column is `target` as-is
-    #     and ascending order is used.
+    #     - `"+"` prefix means ascending order
+    #     - `"-"` prefix means descending order
+    #     - `"^"` prefix means nulls at start
+    #     - `"$"` prefix means nulls at end
     #
-    #   @example String without the leading order mark
-    #     key = Arrow::SortKey.new("count")
-    #     key.target # => "count"
-    #     key.order  # => Arrow::SortOrder::ASCENDING
+    #     If `target` is a Symbol, it is converted to String and used as-is
+    #     (no prefix processing).
     #
-    #   @example String with the "+" leading order mark
-    #     key = Arrow::SortKey.new("+count")
-    #     key.target # => "count"
-    #     key.order  # => Arrow::SortOrder::ASCENDING
+    #   @example String without any prefix
+    #     key = Arrow::SortKey.new("count")
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::ASCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
-    #   @example String with the "-" leading order mark
+    #   @example String with order prefix only
     #     key = Arrow::SortKey.new("-count")
-    #     key.target # => "count"
-    #     key.order  # => Arrow::SortOrder::DESCENDING
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
+    #
+    #   @example String with order and null placement prefixes
+    #     key = Arrow::SortKey.new("-^count")
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_START
     #
-    #   @example Symbol that starts with "-"
+    #   @example String with null placement prefix only
+    #     key = Arrow::SortKey.new("^count")
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::ASCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_START
+    #
+    #   @example Symbol (no prefix processing)
     #     key = Arrow::SortKey.new(:"-count")
-    #     key.target # => "-count"
-    #     key.order  # => Arrow::SortOrder::ASCENDING
+    #     key.target         # => "-count"
+    #     key.order          # => Arrow::SortOrder::ASCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
     # @overload initialize(target, order)
     #
@@ -122,27 +131,54 @@ module Arrow
     #     key = Arrow::SortKey.new("-count", :ascending)
     #     key.target # => "-count"
     #     key.order  # => Arrow::SortOrder::ASCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
     #   @example Order by abbreviated target with Symbol
     #     key = Arrow::SortKey.new("count", :desc)
     #     key.target # => "count"
     #     key.order  # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
     #   @example Order by String
     #     key = Arrow::SortKey.new("count", "descending")
     #     key.target # => "count"
     #     key.order  # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_END
     #
-    #   @example Order by Arrow::SortOrder
-    #     key = Arrow::SortKey.new("count", Arrow::SortOrder::DESCENDING)
+    #   @example Order by Arrow::SortOrder, give null_placement with target
+    #     key = Arrow::SortKey.new("^count", Arrow::SortOrder::DESCENDING)
     #     key.target # => "count"
     #     key.order  # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_START
+    #
+    # @overload initialize(target, order, null_placement)
+    #
+    #   @param target [Symbol, String] The name or dot path of the
+    #     sort column.
+    #
+    #   @param order [Symbol, String, Arrow::SortOrder] How to order
+    #     by this sort key.
+    #
+    #     If this is a Symbol or String, this must be `:ascending`,
+    #     `"ascending"`, `:asc`, `"asc"`, `:descending`,
+    #     `"descending"`, `:desc` or `"desc"`.
+    #
+    #   @param null_placement [Symbol, String, Arrow::NullPlacement]
+    #     Where to place nulls and NaNs. Must be `:at_start`, `"at_start"`,
+    #     `:at_end`, or `"at_end"`.
+    #
+    #   @example With all explicit parameters
+    #     key = Arrow::SortKey.new("count", :desc, :at_start)
+    #     key.target         # => "count"
+    #     key.order          # => Arrow::SortOrder::DESCENDING
+    #     key.null_placement # => Arrow::NullPlacement::AT_START
     #
     # @since 4.0.0
-    def initialize(target, order=nil)
-      target, order = normalize_target(target, order)
+    def initialize(target, order=nil, null_placement=nil)
+      target, order, null_placement = normalize_target(target, order, null_placement)
       order = normalize_order(order) || :ascending
-      initialize_raw(target, order)
+      null_placement = normalize_null_placement(null_placement) || :at_end
+      initialize_raw(target, order, null_placement)
     end
 
     # @return [String] The string representation of this sort key. You
@@ -151,37 +187,66 @@ module Arrow
     #
     # @example Recreate Arrow::SortKey
     #   key = Arrow::SortKey.new("-count")
-    #   key.to_s # => "-count"
+    #   key.to_s # => "-$count"
     #   key == Arrow::SortKey.new(key.to_s) # => true
     #
     # @since 4.0.0
     def to_s
+      result = ""
       if order == SortOrder::ASCENDING
-        "+#{target}"
+        result += "+"
+      else
+        result += "-"
+      end
+      if null_placement == NullPlacement::AT_START
+        result += "^"
       else
-        "-#{target}"
+        result += "$"
       end
+      result += target
+      result
     end
 
     # For backward compatibility
     alias_method :name, :target
 
     private
-    def normalize_target(target, order)
+    # Parse prefix format: [+/-][^/$]column
+    # Examples: -$column, +^column, ^column, -column
+    #
+    # Only strips prefixes if the corresponding parameter is not already set.
+    # This preserves backward compatibility where specifying order explicitly
+    # means the target is used as-is for order prefixes.
+    def normalize_target(target, order, null_placement)
       case target
       when Symbol
-        return target.to_s, order
+        return target.to_s, order, null_placement
       when String
-        return target, order if order
-        if target.start_with?("-")
-          return target[1..-1], order || :descending
-        elsif target.start_with?("+")
-          return target[1..-1], order || :ascending
-        else
-          return target, order
+        remaining = target
+
+        unless order
+          if remaining.start_with?("-")
+            order = :descending
+            remaining = remaining[1..-1]
+          elsif remaining.start_with?("+")
+            order = :ascending
+            remaining = remaining[1..-1]
+          end
+        end
+
+        unless null_placement
+          if remaining.start_with?("^")
+            null_placement = :at_start
+            remaining = remaining[1..-1]
+          elsif remaining.start_with?("$")
+            null_placement = :at_end
+            remaining = remaining[1..-1]
+          end
         end
+
+        return remaining, order, null_placement
       else
-        return target, order
+        return target, order, null_placement
       end
     end
 
@@ -195,5 +260,16 @@ module Arrow
         order
       end
     end
+    
+    def normalize_null_placement(null_placement)
+      case null_placement
+      when :at_end, "at_end"
+        :at_end
+      when :at_start, "at_start"
+        :at_start
+      else
+        null_placement
+      end
+    end
   end
 end

data/lib/arrow/sort-options.rb

@@ -102,8 +102,8 @@ module Arrow
     #     options.sort_keys.collect(&:to_s) # => ["-price"]
     #
     # @since 4.0.0
-    def add_sort_key(target, order=nil)
-      add_sort_key_raw(SortKey.resolve(target, order))
+    def add_sort_key(target, order=nil, null_placement=nil)
+      add_sort_key_raw(SortKey.resolve(target, order, null_placement))
     end
   end
 end

data/lib/arrow/table.rb

@@ -354,60 +354,6 @@ module Arrow
       sliced_table
     end
 
-    # TODO
-    #
-    # @return [Arrow::Table]
-    def merge(other)
-      added_columns = {}
-      removed_columns = {}
-
-      case other
-      when Hash
-        other.each do |name, value|
-          name = name.to_s
-          if value
-            added_columns[name] = ensure_raw_column(name, value)
-          else
-            removed_columns[name] = true
-          end
-        end
-      when Table
-        added_columns = {}
-        other.columns.each do |column|
-          name = column.name
-          added_columns[name] = ensure_raw_column(name, column)
-        end
-      else
-        message = "merge target must be Hash or Arrow::Table: " +
-          "<#{other.inspect}>: #{inspect}"
-        raise ArgumentError, message
-      end
-
-      new_columns = []
-      columns.each do |column|
-        column_name = column.name
-        new_column = added_columns.delete(column_name)
-        if new_column
-          new_columns << new_column
-          next
-        end
-        next if removed_columns.key?(column_name)
-        new_columns << ensure_raw_column(column_name, column)
-      end
-      added_columns.each do |name, new_column|
-        new_columns << new_column
-      end
-      new_fields = []
-      new_arrays = []
-      new_columns.each do |new_column|
-        new_fields << new_column[:field]
-        new_arrays << new_column[:data]
-      end
-      table = self.class.new(new_fields, new_arrays)
-      share_input(table)
-      table
-    end
-
     alias_method :remove_column_raw, :remove_column
     def remove_column(name_or_index)
       case name_or_index

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "24.0.0"
+  VERSION = "25.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 24.0.0
+  version: 25.0.0
 platform: ruby
 authors:
 - The Apache Software Foundation
@@ -259,7 +259,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata:
-  msys2_mingw_dependencies: arrow>=24.0.0
+  msys2_mingw_dependencies: arrow>=25.0.0
 rdoc_options: []
 require_paths:
 - lib
@@ -274,21 +274,21 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- 'system: arrow-glib>=24.0.0: amazon_linux: arrow-glib-devel'
-- 'system: arrow-glib>=24.0.0: amazon_linux: https://packages.apache.org/artifactory/arrow/amazon-linux/%{version}/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=24.0.0: amazon_linux: arrow-glib-devel'
-- 'system: arrow-glib>=24.0.0: centos: arrow-glib-devel'
-- 'system: arrow-glib>=24.0.0: centos: https://packages.apache.org/artifactory/arrow/centos/%{major_version}-stream/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=24.0.0: centos: arrow-glib-devel'
-- 'system: arrow-glib>=24.0.0: conda: arrow-c-glib'
-- 'system: arrow-glib>=24.0.0: debian: libarrow-glib-dev'
-- 'system: arrow-glib>=24.0.0: debian: https://packages.apache.org/artifactory/arrow/%{distribution}/apache-arrow-apt-source-latest-%{code_name}.deb'
-- 'system: arrow-glib>=24.0.0: debian: libarrow-glib-dev'
-- 'system: arrow-glib>=24.0.0: fedora: libarrow-glib-devel'
-- 'system: arrow-glib>=24.0.0: homebrew: apache-arrow-glib'
-- 'system: arrow-glib>=24.0.0: rhel: arrow-glib-devel'
-- 'system: arrow-glib>=24.0.0: rhel: https://packages.apache.org/artifactory/arrow/almalinux/%{major_version}/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=24.0.0: rhel: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: amazon_linux: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: amazon_linux: https://packages.apache.org/artifactory/arrow/amazon-linux/%{version}/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=25.0.0: amazon_linux: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: centos: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: centos: https://packages.apache.org/artifactory/arrow/centos/%{major_version}-stream/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=25.0.0: centos: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: conda: arrow-c-glib'
+- 'system: arrow-glib>=25.0.0: debian: libarrow-glib-dev'
+- 'system: arrow-glib>=25.0.0: debian: https://packages.apache.org/artifactory/arrow/%{distribution}/apache-arrow-apt-source-latest-%{code_name}.deb'
+- 'system: arrow-glib>=25.0.0: debian: libarrow-glib-dev'
+- 'system: arrow-glib>=25.0.0: fedora: libarrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: homebrew: apache-arrow-glib'
+- 'system: arrow-glib>=25.0.0: rhel: arrow-glib-devel'
+- 'system: arrow-glib>=25.0.0: rhel: https://packages.apache.org/artifactory/arrow/almalinux/%{major_version}/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=25.0.0: rhel: arrow-glib-devel'
 rubygems_version: 3.6.7
 specification_version: 4
 summary: Red Arrow is the Ruby bindings of Apache Arrow