red-arrow 5.0.0 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcc31d7ec789dfa277963fda5b3710e7eec2451b6fbe425282638e48e684b95f
-  data.tar.gz: 00bfbf2e0357861f7247c42c4faae6a72d9adfca745ad1b3298f549997847158
+  metadata.gz: c553e573f87398da7d492e8339684bb86b73d1fff02b56cff3277bb87d96a4cd
+  data.tar.gz: 9b97be2878369da0bed9ac8621330926b8a95144d833de6798858a210f88c92a
 SHA512:
-  metadata.gz: 8e60e9a4ce4d379f7f1d9387154ae4d9fe4ec91ffe304647966d33d52e380950cc0a8a5f025164466526feb5198638139bb7564f2066fd93b7c21dcd697e07ca
-  data.tar.gz: 140a8f087eeb42baae703eee677c97261c6b30317c0f72ce49231a117b2e32f34175bad71f913bdd6f940abd8097c77f968b27780cecbfca062be60ac87d09ec
+  metadata.gz: 0bc3526645457a551e25621ba3efaa52a3ad7398e8216c34440651beab58886ec8b52804b7afe227f953b727871980a2ac9a9b1153776499b48cd790ae476304
+  data.tar.gz: 9f3032d5a54a9b0b911edd273916ce68f140c50753d13cb88b950166ea771138f432d738a3a694f44bfff985fcf8329457e97712dca10a7f8a8430d3d8cb82f6

data/README.md

@@ -50,3 +50,26 @@ table = Arrow::Table.load("/dev/shm/data.arrow")
 # Process data in table
 table.save("/dev/shm/data-processed.arrow")
 ```
+
+## Development
+
+Note that you need to install Apache Arrow C++/GLib at master before preparing Red Arrow. See also:
+
+  * For Apache Arrow C++: https://arrow.apache.org/docs/developers/cpp/building.html
+  * For Apache Arrow GLib: https://github.com/apache/arrow/blob/master/c_glib/README.md
+
+```console
+$ cd ruby/red-arrow
+$ bundle install
+$ bundle exec rake test
+```
+
+### For macOS with Homebrew
+
+```console
+$ cd ruby/red-arrow
+$ bundle install
+$ brew install apache-arrow --head
+$ brew install apache-arrow-glib --head
+$ bundle exec rake test
+```

data/ext/arrow/converters.cpp

@@ -30,6 +30,11 @@ namespace red_arrow {
     return struct_array_value_converter_->convert(array, i);
   }
 
+  VALUE ArrayValueConverter::convert(const arrow::MapArray& array,
+                                     const int64_t i) {
+    return map_array_value_converter_->convert(array, i);
+  }
+
   VALUE ArrayValueConverter::convert(const arrow::UnionArray& array,
                                      const int64_t i) {
     return union_array_value_converter_->convert(array, i);

data/ext/arrow/converters.hpp

@@ -29,6 +29,7 @@
 namespace red_arrow {
   class ListArrayValueConverter;
   class StructArrayValueConverter;
+  class MapArrayValueConverter;
   class UnionArrayValueConverter;
   class DictionaryArrayValueConverter;
 
@@ -38,16 +39,19 @@ namespace red_arrow {
       : decimal_buffer_(),
         list_array_value_converter_(nullptr),
         struct_array_value_converter_(nullptr),
+        map_array_value_converter_(nullptr),
         union_array_value_converter_(nullptr),
         dictionary_array_value_converter_(nullptr) {
     }
 
     inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter,
                                          StructArrayValueConverter* struct_array_value_converter,
+                                         MapArrayValueConverter* map_array_value_converter,
                                          UnionArrayValueConverter* union_array_value_converter,
                                          DictionaryArrayValueConverter* dictionary_array_value_converter) {
       list_array_value_converter_ = list_array_value_converter;
       struct_array_value_converter_ = struct_array_value_converter;
+      map_array_value_converter_ = map_array_value_converter;
       union_array_value_converter_ = union_array_value_converter;
       dictionary_array_value_converter_ = dictionary_array_value_converter;
     }
@@ -204,6 +208,9 @@ namespace red_arrow {
     VALUE convert(const arrow::StructArray& array,
                   const int64_t i);
 
+    VALUE convert(const arrow::MapArray& array,
+                  const int64_t i);
+
     VALUE convert(const arrow::UnionArray& array,
                   const int64_t i);
 
@@ -234,6 +241,7 @@ namespace red_arrow {
     std::string decimal_buffer_;
     ListArrayValueConverter* list_array_value_converter_;
     StructArrayValueConverter* struct_array_value_converter_;
+    MapArrayValueConverter* map_array_value_converter_;
     UnionArrayValueConverter* union_array_value_converter_;
     DictionaryArrayValueConverter* dictionary_array_value_converter_;
   };
@@ -294,6 +302,7 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
+    VISIT(Map)
     VISIT(SparseUnion)
     VISIT(DenseUnion)
     VISIT(Dictionary)
@@ -399,6 +408,7 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
+    VISIT(Map)
     VISIT(SparseUnion)
     VISIT(DenseUnion)
     VISIT(Dictionary)
@@ -431,6 +441,117 @@ namespace red_arrow {
     VALUE result_;
   };
 
+  class MapArrayValueConverter : public arrow::ArrayVisitor {
+  public:
+    explicit MapArrayValueConverter(ArrayValueConverter* converter)
+      : array_value_converter_(converter),
+        offset_(0),
+        length_(0),
+        values_(Qnil) {}
+
+    VALUE convert(const arrow::MapArray& array,
+                  const int64_t index) {
+      auto key_array = array.keys().get();
+      auto item_array = array.items().get();
+      auto offset_keep = offset_;
+      auto length_keep = length_;
+      auto values_keep = values_;
+      offset_ = array.value_offset(index);
+      length_ = array.value_length(index);
+      auto keys = rb_ary_new_capa(length_);
+      values_ = keys;
+      check_status(key_array->Accept(this),
+                   "[raw-records][map-array][keys]");
+      auto items = rb_ary_new_capa(length_);
+      values_ = items;
+      check_status(item_array->Accept(this),
+                   "[raw-records][map-array][items]");
+      auto map = rb_hash_new();
+      auto n = RARRAY_LEN(keys);
+      auto raw_keys = RARRAY_CONST_PTR(keys);
+      auto raw_items = RARRAY_CONST_PTR(items);
+      for (long i = 0; i < n; ++i) {
+        rb_hash_aset(map, raw_keys[i], raw_items[i]);
+      }
+      offset_ = offset_keep;
+      length_ = length_keep;
+      values_ = values_keep;
+      return map;
+    }
+
+#define VISIT(TYPE)                                                     \
+    arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
+      return visit_value(array);                                        \
+    }
+
+    VISIT(Null)
+    VISIT(Boolean)
+    VISIT(Int8)
+    VISIT(Int16)
+    VISIT(Int32)
+    VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    // TODO
+    // VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    // TODO
+    // VISIT(Interval)
+    VISIT(List)
+    VISIT(Struct)
+    VISIT(Map)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    VISIT(Decimal256)
+    // TODO
+    // VISIT(Extension)
+
+#undef VISIT
+
+  private:
+    template <typename ArrayType>
+    inline VALUE convert_value(const ArrayType& array,
+                               const int64_t i) {
+      return array_value_converter_->convert(array, i);
+    }
+
+    template <typename ArrayType>
+    arrow::Status visit_value(const ArrayType& array) {
+      if (array.null_count() > 0) {
+        for (int64_t i = 0; i < length_; ++i) {
+          auto value = Qnil;
+          if (!array.IsNull(i + offset_)) {
+            value = convert_value(array, i + offset_);
+          }
+          rb_ary_push(values_, value);
+        }
+      } else {
+        for (int64_t i = 0; i < length_; ++i) {
+          rb_ary_push(values_, convert_value(array, i + offset_));
+        }
+      }
+      return arrow::Status::OK();
+    }
+
+    ArrayValueConverter* array_value_converter_;
+    int32_t offset_;
+    int32_t length_;
+    VALUE values_;
+  };
+
   class UnionArrayValueConverter : public arrow::ArrayVisitor {
   public:
     explicit UnionArrayValueConverter(ArrayValueConverter* converter)
@@ -492,6 +613,7 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
+    VISIT(Map)
     VISIT(SparseUnion)
     VISIT(DenseUnion)
     VISIT(Dictionary)
@@ -617,6 +739,7 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
+    VISIT(Map)
     VISIT(SparseUnion)
     VISIT(DenseUnion)
     VISIT(Dictionary)
@@ -645,11 +768,13 @@ namespace red_arrow {
       : array_value_converter_(),
         list_array_value_converter_(&array_value_converter_),
         struct_array_value_converter_(&array_value_converter_),
+        map_array_value_converter_(&array_value_converter_),
         union_array_value_converter_(&array_value_converter_),
         dictionary_array_value_converter_(&array_value_converter_) {
       array_value_converter_.
         set_sub_value_converters(&list_array_value_converter_,
                                  &struct_array_value_converter_,
+                                 &map_array_value_converter_,
                                  &union_array_value_converter_,
                                  &dictionary_array_value_converter_);
     }
@@ -663,6 +788,7 @@ namespace red_arrow {
     ArrayValueConverter array_value_converter_;
     ListArrayValueConverter list_array_value_converter_;
     StructArrayValueConverter struct_array_value_converter_;
+    MapArrayValueConverter map_array_value_converter_;
     UnionArrayValueConverter union_array_value_converter_;
     DictionaryArrayValueConverter dictionary_array_value_converter_;
   };

data/ext/arrow/extconf.rb

@@ -25,6 +25,19 @@ if arrow_pkg_config_path
   ENV["PKG_CONFIG_PATH"] = pkg_config_paths.join(File::PATH_SEPARATOR)
 end
 
+checking_for(checking_message("Homebrew")) do
+  platform = NativePackageInstaller::Platform.detect
+  if platform.is_a?(NativePackageInstaller::Platform::Homebrew)
+    openssl_prefix = `brew --prefix openssl@1.1`.chomp
+    unless openssl_prefix.empty?
+      PKGConfig.add_path("#{openssl_prefix}/lib/pkgconfig")
+    end
+    true
+  else
+    false
+  end
+end
+
 unless required_pkg_config_package([
                                      "arrow",
                                      Arrow::Version::MAJOR,

data/ext/arrow/raw-records.cpp

@@ -100,6 +100,7 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
+      VISIT(Map)
       VISIT(SparseUnion)
       VISIT(DenseUnion)
       VISIT(Dictionary)

data/ext/arrow/values.cpp

@@ -81,6 +81,7 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
+      VISIT(Map)
       VISIT(SparseUnion)
       VISIT(DenseUnion)
       VISIT(Dictionary)

data/lib/arrow/aggregate-node-options.rb

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class AggregateNodeOptions
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          aggregations = value[:aggregations]
+          return nil if aggregations.nil?
+          keys = value[:keys]
+          new(aggregations, keys)
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/aggregation.rb

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Aggregation
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Hash
+          function = value[:function]
+          return nil if function.nil?
+          function = function.to_s if function.is_a?(Symbol)
+          return nil unless function.is_a?(String)
+          # TODO: Improve this when we have non hash based aggregate function
+          function = "hash_#{function}" unless function.start_with?("hash_")
+          options = value[:options]
+          input = value[:input]
+          return nil if input.nil?
+          output = value[:output]
+          if output.nil?
+            normalized_function = function.gsub(/\Ahash_/, "")
+            output = "#{normalized_function}(#{input})"
+          end
+          new(function, options, input, output)
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/array-builder.rb

@@ -58,6 +58,11 @@ module Arrow
             builder: StringArrayBuilder.new,
             detected: true,
           }
+        when Symbol
+          {
+            builder: StringDictionaryArrayBuilder.new,
+            detected: true,
+          }
         when Float
           {
             builder: DoubleArrayBuilder.new,

data/lib/arrow/binary-dictionary-array-builder.rb

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class BinaryDictionaryArrayBuilder
+    include SymbolValuesAppendable
+
+    private
+    def create_values_array_builder
+      BinaryArrayBuilder.new
+    end
+  end
+end

data/lib/arrow/column-containable.rb

@@ -27,6 +27,17 @@ module Arrow
       columns.each(&block)
     end
 
+    # @overload [](name)
+    #   Find a column that has the given name.
+    #
+    #   @param name [String, Symbol] The column name to be found.
+    #   @return [Column] The found column.
+    #
+    # @overload [](index)
+    #   Find the `index`-th column.
+    #
+    #   @param index [Integer] The index to be found.
+    #   @return [Column] The found column.
     def find_column(name_or_index)
       case name_or_index
       when String, Symbol
@@ -40,9 +51,97 @@ module Arrow
         return nil if index < 0 or index >= n_columns
         Column.new(self, index)
       else
-        message = "column name or index must be String, Symbol or Integer"
+        message = "column name or index must be String, Symbol or Integer: "
+        message << name_or_index.inspect
         raise ArgumentError, message
       end
     end
+
+    # Selects columns that are selected by `selectors` and/or `block`
+    # and creates a new container only with the selected columns.
+    #
+    # @param selectors [Array<String, Symbol, Integer, Range>]
+    #   If a selector is `String`, `Symbol` or `Integer`, the selector
+    #   selects a column by {#find_column}.
+    #
+    #   If a selector is `Range`, the selector selects columns by `::Array#[]`.
+    # @yield [column] Gives a column to the block to select columns.
+    #   This uses `::Array#select`.
+    # @yieldparam column [Column] A target column.
+    # @yieldreturn [Boolean] Whether the given column is selected or not.
+    # @return [self.class] The newly created container that only has selected
+    #   columns.
+    def select_columns(*selectors, &block)
+      if selectors.empty?
+        return to_enum(__method__) unless block_given?
+        selected_columns = columns.select(&block)
+      else
+        selected_columns = []
+        selectors.each do |selector|
+          case selector
+          when Range
+            selected_columns.concat(columns[selector])
+          else
+            column = find_column(selector)
+            if column.nil?
+              case selector
+              when String, Symbol
+                message = "unknown column: #{selector.inspect}: #{inspect}"
+                raise KeyError.new(message)
+              else
+                message = "out of index (0..#{n_columns - 1}): "
+                message << "#{selector.inspect}: #{inspect}"
+                raise IndexError.new(message)
+              end
+            end
+            selected_columns << column
+          end
+        end
+        selected_columns = selected_columns.select(&block) if block_given?
+      end
+      self.class.new(selected_columns)
+    end
+
+    # @overload [](name)
+    #   Find a column that has the given name.
+    #
+    #   @param name [String, Symbol] The column name to be found.
+    #   @return [Column] The found column.
+    #   @see #find_column
+    #
+    # @overload [](index)
+    #   Find the `index`-th column.
+    #
+    #   @param index [Integer] The index to be found.
+    #   @return [Column] The found column.
+    #   @see #find_column
+    #
+    # @overload [](range)
+    #   Selects columns that are in `range` and creates a new container
+    #   only with the selected columns.
+    #
+    #   @param range [Range] The range to be selected.
+    #   @return [self.class] The newly created container that only has selected
+    #     columns.
+    #   @see #select_columns
+    #
+    # @overload [](selectors)
+    #   Selects columns that are selected by `selectors` and creates a
+    #   new container only with the selected columns.
+    #
+    #   @param selectors [Array] The selectors that are used to select columns.
+    #   @return [self.class] The newly created container that only has selected
+    #     columns.
+    #   @see #select_columns
+    def [](selector)
+      case selector
+      when ::Array
+        select_columns(*selector)
+      when Range
+        select_columns(selector)
+      else
+        find_column(selector)
+      end
+    end
   end
 end

data/lib/arrow/datum.rb

@@ -21,6 +21,8 @@ module Arrow
       # @api private
       def try_convert(value)
         case value
+        when Table
+          TableDatum.new(value)
         when Array
           ArrayDatum.new(value)
         when ChunkedArray

data/lib/arrow/expression.rb

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Expression
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Symbol
+          FieldExpression.new(value.to_s)
+        when ::Array
+          function_name, *arguments = value
+          case function_name
+          when String, Symbol
+            function_name = function_name.to_s
+          else
+            return nil
+          end
+          if arguments.last.is_a?(FunctionOptions)
+            options = arguments.pop
+          else
+            options = nil
+          end
+          CallExpression.new(function_name, arguments, options)
+        else
+          datum = Datum.try_convert(value)
+          return nil if datum.nil?
+          LiteralExpression.new(datum)
+        end
+      end
+    end
+  end
+end

data/lib/arrow/file-system.rb

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class FileSystem
+    alias_method :open_output_stream_raw, :open_output_stream
+    def open_output_stream(path)
+      stream = open_output_stream_raw(path)
+      if block_given?
+        begin
+          yield(stream)
+        ensure
+          stream.close
+        end
+      else
+        stream
+      end
+    end
+  end
+end