red-arrow 11.0.0 → 13.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23f72b7016d780c208dd8f8cbc627becce73e25754f83a62a4a4d3f4bb60a5e3
-  data.tar.gz: 23f57e383e26d322e9fa81efed1e8e5545bd5b1160075f4a0b8553efda98af93
+  metadata.gz: dcbc004b8b61405c6e1143428ca37aac1be3de4d31d73f0a235f9c2e8d7bf5bb
+  data.tar.gz: 508ba001225d07eae018b7552987c6102a8b59c98f4d717b4e88ad1a188d84b3
 SHA512:
-  metadata.gz: 2d157d2d56dbca00a2f4a0eadb831a19a7a7124c3c3c8c7675536e4597f2a1447f047eec48654256eff3148457d09f1d3e375a095cf152051ef6603cd8bf25bf
-  data.tar.gz: 958f320a92981a6ed7f84fcab92497ac1038a038de302419e9f57f83e3ca4d02d682ec05a129ac63c9ed6e613725619839b46cc589938a56573e88182a8002d4
+  metadata.gz: 5cbeae509369615824b6c5d41d8f97d4b166ca99d9b1dc3fba4207e726ac22a0147027cb531761ba91245df7d71bb02ab370221ec72359470149ace65b2a01f7
+  data.tar.gz: 2043d58148d1370789ad6b75ed605b7d3a78c1094f2250a09b0857a43d24f37a83a5706b3b96b41dfa7f0dafa22179dde63a4e991e58e85f19002759cfed52b8

data/README.md

@@ -25,9 +25,9 @@ Red Arrow is the Ruby bindings of Apache Arrow. Red Arrow is based on GObject In
 
 [GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection) is a middleware for language bindings of C library. GObject Introspection can generate language bindings automatically at runtime.
 
-Red Arrow uses [Apache Arrow GLib](https://github.com/apache/arrow/tree/master/c_glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Arrow.
+Red Arrow uses [Apache Arrow GLib](https://github.com/apache/arrow/tree/main/c_glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Arrow.
 
-Apache Arrow GLib is a C wrapper for [Apache Arrow C++](https://github.com/apache/arrow/tree/master/cpp). GObject Introspection can't use Apache Arrow C++ directly. Apache Arrow GLib is a bridge between Apache Arrow C++ and GObject Introspection.
+Apache Arrow GLib is a C wrapper for [Apache Arrow C++](https://github.com/apache/arrow/tree/main/cpp). GObject Introspection can't use Apache Arrow C++ directly. Apache Arrow GLib is a bridge between Apache Arrow C++ and GObject Introspection.
 
 gobject-introspection gem is a Ruby bindings of GObject Introspection. Red Arrow uses GObject Introspection via gobject-introspection gem.
 
@@ -56,7 +56,7 @@ table.save("/dev/shm/data-processed.arrow")
 Note that you need to install Apache Arrow C++/GLib at master before preparing Red Arrow. See also:
 
   * For Apache Arrow C++: https://arrow.apache.org/docs/developers/cpp/building.html
-  * For Apache Arrow GLib: https://github.com/apache/arrow/blob/master/c_glib/README.md
+  * For Apache Arrow GLib: https://github.com/apache/arrow/blob/main/c_glib/README.md
 
 ```console
 $ cd ruby/red-arrow

data/ext/arrow/arrow.cpp

@@ -43,6 +43,26 @@ namespace red_arrow {
     VALUE month;
     VALUE nanosecond;
   }
+
+  void
+  record_batch_reader_mark(gpointer object)
+  {
+    auto reader = GARROW_RECORD_BATCH_READER(object);
+    auto sources = garrow_record_batch_reader_get_sources(reader);
+    for (auto source = sources; sources; sources = g_list_next(sources)) {
+      rbgobj_gc_mark_instance(source->data);
+    }
+  }
+
+  void
+  execute_plan_mark(gpointer object)
+  {
+    auto plan = GARROW_EXECUTE_PLAN(object);
+    auto nodes = garrow_execute_plan_get_nodes(plan);
+    for (auto node = nodes; nodes; nodes = g_list_next(nodes)) {
+      rbgobj_gc_mark_instance(node->data);
+    }
+  }
 }
 
 extern "C" void Init_arrow() {
@@ -93,4 +113,9 @@ extern "C" void Init_arrow() {
   red_arrow::symbols::millisecond = ID2SYM(rb_intern("millisecond"));
   red_arrow::symbols::month = ID2SYM(rb_intern("month"));
   red_arrow::symbols::nanosecond = ID2SYM(rb_intern("nanosecond"));
+
+  rbgobj_register_mark_func(GARROW_TYPE_RECORD_BATCH_READER,
+                            red_arrow::record_batch_reader_mark);
+  rbgobj_register_mark_func(GARROW_TYPE_EXECUTE_PLAN,
+                            red_arrow::execute_plan_mark);
 }

data/ext/arrow/converters.hpp

@@ -685,25 +685,21 @@ namespace red_arrow {
   private:
     template <typename ArrayType>
     inline void convert_value(const ArrayType& array) {
-      auto result = rb_hash_new();
       if (array.IsNull(index_)) {
-        rb_hash_aset(result, field_name_, Qnil);
+        result_ = RUBY_Qnil;
       } else {
-        rb_hash_aset(result,
-                     field_name_,
-                     array_value_converter_->convert(array, index_));
+        result_ = array_value_converter_->convert(array, index_);
       }
-      result_ = result;
     }
 
-    uint8_t compute_field_index(const arrow::UnionArray& array,
-                                arrow::UnionType* type,
-                                const char* tag) {
+    int8_t compute_child_id(const arrow::UnionArray& array,
+                            arrow::UnionType* type,
+                            const char* tag) {
       const auto type_code = array.raw_type_codes()[index_];
       if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
-        const auto field_id = type->child_ids()[type_code];
-        if (field_id >= 0) {
-          return field_id;
+        const auto child_id = type->child_ids()[type_code];
+        if (child_id >= 0) {
+          return child_id;
         }
       }
       check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
@@ -715,36 +711,25 @@ namespace red_arrow {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-sparse-array]";
-      const auto index = compute_field_index(array, type, tag);
-      const auto field = type->field(index).get();
-      const auto& field_name = field->name();
-      const auto field_name_keep = field_name_;
-      field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto field_array = array.field(index).get();
+      const auto child_id = compute_child_id(array, type, tag);
+      const auto field_array = array.field(child_id).get();
       check_status(field_array->Accept(this), tag);
-      field_name_ = field_name_keep;
     }
 
     void convert_dense(const arrow::DenseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-dense-array]";
-      const auto index = compute_field_index(array, type, tag);
-      const auto field = type->field(index).get();
-      const auto& field_name = field->name();
-      const auto field_name_keep = field_name_;
-      field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto field_array = array.field(index);
+      const auto child_id = compute_child_id(array, type, tag);
+      const auto field_array = array.field(child_id);
       const auto index_keep = index_;
       index_ = array.value_offset(index_);
       check_status(field_array->Accept(this), tag);
       index_ = index_keep;
-      field_name_ = field_name_keep;
     }
 
     ArrayValueConverter* array_value_converter_;
     int64_t index_;
-    VALUE field_name_;
     VALUE result_;
   };
 

data/ext/arrow/extconf.rb

@@ -41,9 +41,8 @@ end
 unless required_pkg_config_package([
                                      "arrow",
                                      Arrow::Version::MAJOR,
-                                     Arrow::Version::MINOR,
-                                     Arrow::Version::MICRO,
                                    ],
+                                   conda: "libarrow",
                                    debian: "libarrow-dev",
                                    fedora: "libarrow-devel",
                                    homebrew: "apache-arrow",
@@ -58,6 +57,7 @@ unless required_pkg_config_package([
                                      Arrow::Version::MINOR,
                                      Arrow::Version::MICRO,
                                    ],
+                                   conda: "arrow-c-glib",
                                    debian: "libarrow-glib-dev",
                                    fedora: "libarrow-glib-devel",
                                    homebrew: "apache-arrow-glib",

data/lib/arrow/array-computable.rb

@@ -29,6 +29,19 @@ module Arrow
       unique.values
     end
 
+    # Finds the index of the first occurrence of a given value.
+    #
+    # @param value [Object] The value to be compared.
+    #
+    # @return [Integer] The index of the first occurrence of a given
+    #   value on found, -1 on not found.
+    #
+    # @since 12.0.0
+    def index(value)
+      value = Scalar.resolve(value, value_data_type)
+      compute("index", options: {value: value}).value
+    end
+
     private
     def compute(name, options: nil)
       Function.find(name).execute([self], options).value

data/lib/arrow/chunked-array.rb

@@ -29,7 +29,11 @@ module Arrow
     end
 
     def to_arrow_array
-      combine
+      if n_chunks.zero?
+        value_data_type.build_array([])
+      else
+        combine
+      end
     end
 
     def to_arrow_chunked_array

data/lib/arrow/data-type.rb

@@ -199,5 +199,14 @@ module Arrow
       args.unshift(self) unless builder_class.buildable?(args)
       builder_class.build(*args)
     end
+
+    # @return [Arrow::Scalar} A corresponding {Arrow::Scalar} class
+    #   for this data type.
+    #
+    # @since 12.0.0
+    def scalar_class
+      base_name = self.class.name.gsub(/DataType\z/, "")
+      ::Arrow.const_get("#{base_name}Scalar")
+    end
   end
 end

data/lib/arrow/dense-union-array-builder.rb

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DenseUnionArrayBuilder
+    alias_method :append_value_raw, :append_value
+
+    # @overload append_value
+    #
+    #   Starts appending an union record. You need to append values of
+    #   fields.
+    #
+    # @overload append_value(value)
+    #
+    #   Appends an union record including values of fields.
+    #
+    #   @param value [nil, Hash] The union record value.
+    #
+    #     If this is `nil`, the union record is null.
+    #
+    #     If this is `Hash`, it's  values of fields.
+    #
+    # @since 12.0.0
+    def append_value(value)
+      if value.nil?
+        append_null
+      else
+        key = value.keys[0]
+        child_info = child_infos[key]
+        append_value_raw(child_info[:id])
+        child_info[:builder].append(value.values[0])
+      end
+    end
+  end
+end

data/lib/arrow/dense-union-array.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DenseUnionArray
+    def get_value(i)
+      child_id = get_child_id(i)
+      field = get_field(child_id)
+      field[get_value_offset(i)]
+    end
+  end
+end

data/lib/arrow/expression.rb

@@ -31,10 +31,14 @@ module Arrow
           else
             return nil
           end
+          options = nil
           if arguments.last.is_a?(FunctionOptions)
             options = arguments.pop
-          else
-            options = nil
+          elsif arguments.last.is_a?(Hash)
+            function = Function.find(function_name)
+            if function
+              options = function.resolve_options(arguments.pop)
+            end
           end
           CallExpression.new(function_name, arguments, options)
         else

data/lib/arrow/function.rb

@@ -24,7 +24,6 @@ module Arrow
     end
     alias_method :call, :execute
 
-    private
     def resolve_options(options)
       return nil if options.nil?
       return options if options.is_a?(FunctionOptions)

data/lib/arrow/loader.rb

@@ -70,6 +70,8 @@ module Arrow
       require "arrow/decimal256-array"
       require "arrow/decimal256-array-builder"
       require "arrow/decimal256-data-type"
+      require "arrow/dense-union-array"
+      require "arrow/dense-union-array-builder"
       require "arrow/dense-union-data-type"
       require "arrow/dictionary-array"
       require "arrow/dictionary-data-type"
@@ -109,6 +111,8 @@ module Arrow
       require "arrow/sort-key"
       require "arrow/sort-options"
       require "arrow/source-node-options"
+      require "arrow/sparse-union-array"
+      require "arrow/sparse-union-array-builder"
       require "arrow/sparse-union-data-type"
       require "arrow/string-dictionary-array-builder"
       require "arrow/string-array-builder"
@@ -134,6 +138,7 @@ module Arrow
       require "arrow/timestamp-array"
       require "arrow/timestamp-array-builder"
       require "arrow/timestamp-data-type"
+      require "arrow/union-array-builder"
       require "arrow/writable"
     end
 

data/lib/arrow/record-batch-file-reader.rb

@@ -20,6 +20,8 @@ module Arrow
     include Enumerable
 
     def each
+      return to_enum(__method__) {n_record_batches} unless block_given?
+
       n_record_batches.times do |i|
         yield(get_record_batch(i))
       end

data/lib/arrow/record-batch-stream-reader.rb

@@ -20,6 +20,8 @@ module Arrow
     include Enumerable
 
     def each
+      return to_enum(__method__) unless block_given?
+
       loop do
         record_batch = next_record_batch
         break if record_batch.nil?

data/lib/arrow/scalar.rb

@@ -17,6 +17,73 @@
 
 module Arrow
   class Scalar
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when self
+          value
+        when true, false
+          BooleanScalar.new(value)
+        when Symbol, String
+          StringScalar.new(value.to_s)
+        when Integer
+          Int64Scalar.new(value)
+        when Float
+          DoubleScalar.new(value)
+        else
+          nil
+        end
+      end
+
+      # Ensure returning suitable {Arrow::Scalar}.
+      #
+      # @overload resolve(scalar)
+      #
+      #   Returns the given scalar itself. This is convenient to
+      #   use this method as {Arrow::Scalar} converter.
+      #
+      #   @param scalar [Arrow::Scalar] The scalar.
+      #
+      #   @return [Arrow::Scalar] The given scalar itself.
+      #
+      # @overload resolve(value)
+      #
+      #   Creates a suitable scalar from the given value. For example,
+      #   you can create {Arrow::BooleanScalar} from `true`.
+      #
+      #   @param value [Object] The value.
+      #
+      #   @return [Arrow::Scalar] A suitable {Arrow::Scalar} for `value`.
+      #
+      # @overload resolve(value, data_type)
+      #
+      #   Creates a scalar of `data_type.scalar_class` from the given
+      #   value. For example, you can create {Arrow::Int32Scalar} from
+      #   `29` and {Arrow::Int32DataType}.
+      #
+      #   @param value [Object] The value.
+      #
+      #   @param data_type [Arrow::DataType] The {Arrow::DataType} to
+      #     decide the returned scalar class.
+      #
+      #   @return [Arrow::Scalar] A suitable {Arrow::Scalar} for `value`.
+      #
+      # @since 12.0.0
+      def resolve(value, data_type=nil)
+        return try_convert(value) if data_type.nil?
+
+        data_type = DataType.resolve(data_type)
+        scalar_class = data_type.scalar_class
+        case value
+        when Scalar
+          return value if value.class == scalar_class
+          value = value.value
+        end
+        scalar_class.new(value)
+      end
+    end
+
     # @param other [Arrow::Scalar] The scalar to be compared.
     # @param options [Arrow::EqualOptions, Hash] (nil)
     #   The options to custom how to compare.

data/lib/arrow/slicer.rb

@@ -162,6 +162,40 @@ module Arrow
       def reject(&block)
         RejectCondition.new(@column, block)
       end
+
+      def end_with?(substring, ignore_case: false)
+        MatchSubstringFamilyCondition.new("ends_with",
+                                          @column, substring, ignore_case)
+      end
+
+      def match_like?(pattern, ignore_case: false)
+        MatchSubstringFamilyCondition.new("match_like",
+                                          @column, pattern, ignore_case)
+      end
+
+      def match_substring?(pattern, ignore_case: nil)
+        case pattern
+        when String
+          ignore_case = false if ignore_case.nil?
+          MatchSubstringFamilyCondition.new("match_substring",
+                                            @column, pattern, ignore_case)
+        when Regexp
+          ignore_case = pattern.casefold? if ignore_case.nil?
+          MatchSubstringFamilyCondition.new("match_substring_regex",
+                                            @column,
+                                            pattern.source,
+                                            ignore_case)
+        else
+          message =
+             "pattern must be either String or Regexp: #{pattern.inspect}"
+          raise ArgumentError, message
+        end
+      end
+
+      def start_with?(substring, ignore_case: false)
+        MatchSubstringFamilyCondition.new("starts_with",
+                                          @column, substring, ignore_case)
+      end
     end
 
     class NotColumnCondition < Condition
@@ -351,5 +385,32 @@ module Arrow
         BooleanArray.new(raw_array)
       end
     end
+
+    class MatchSubstringFamilyCondition < Condition
+      def initialize(function, column, pattern, ignore_case, invert: false)
+        @function = function
+        @column = column
+        @options = MatchSubstringOptions.new
+        @options.pattern = pattern
+        @options.ignore_case = ignore_case
+        @invert = invert
+      end
+
+      def !@
+        MatchSubstringFamilyCondition.new(@function,
+                                          @column,
+                                          @options.pattern,
+                                          @options.ignore_case?,
+                                          invert: !@invert)
+      end
+
+      def evaluate
+        datum = Function.find(@function).execute([@column.data], @options)
+        if @invert
+          datum = Function.find("invert").execute([datum])
+        end
+        datum.value
+      end
+    end
   end
 end

data/lib/arrow/sparse-union-array-builder.rb

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class SparseUnionArrayBuilder
+    alias_method :append_value_raw, :append_value
+
+    # @overload append_value
+    #
+    #   Starts appending an union record. You need to append values of
+    #   fields.
+    #
+    # @overload append_value(value)
+    #
+    #   Appends an union record including values of fields.
+    #
+    #   @param value [nil, Hash] The union record value.
+    #
+    #     If this is `nil`, the union record is null.
+    #
+    #     If this is `Hash`, it's  values of fields.
+    #
+    # @since 12.0.0
+    def append_value(value)
+      if value.nil?
+        append_null
+      else
+        key = value.keys[0]
+        child_info = child_infos[key]
+        append_value_raw(child_info[:id])
+        child_infos.each do |child_key, child_info|
+          builder = child_info[:builder]
+          if child_key == key
+            builder.append(value.values[0])
+          else
+            builder.append_null
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow/sparse-union-array.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class SparseUnionArray
+    def get_value(i)
+      child_id = get_child_id(i)
+      field = get_field(child_id)
+      field[i]
+    end
+  end
+end

data/lib/arrow/struct-array-builder.rb

@@ -110,11 +110,6 @@ module Arrow
       end
     end
 
-    alias_method :append_null_raw, :append_null
-    def append_null
-      append_null_raw
-    end
-
     # @since 0.12.0
     def append(*values)
       if values.empty?