red-arrow 22.0.0 → 23.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae7465055f36320c48284183aecfd96123b7f8bf38624fb666f733eb2a74e69f
-  data.tar.gz: 88f7d6849a9ff3f4ca016130ac6599cd1e9a14178127fe1cf6c240ae067d3d30
+  metadata.gz: a5d8f71916813b1efc347972a6ad663bf84561dba641e61d8367e7bec67adbca
+  data.tar.gz: 4667c7515eb8662a734907348851ae557f982fb6ad534f6f18a3e8aa10dce15f
 SHA512:
-  metadata.gz: 4e584fc27a4a955e94ba4316248078a3bac66711403cccda44823d236cc65f6d9c70fd8c9495ec3da85e7b29ee68bcb4e51fbbb675d950232cbb73ed64addbbd
-  data.tar.gz: 9b807446da0b8a5536a20bf5d662036e6d0fb4f1e5594c31527dc59b06dada081196229d4171cc481e3138bf361f5e09ed62655b9267c96b1dac935c99295054
+  metadata.gz: c18d5f0d79959d48fd69719350fa31636853c073a3e11743b52654eacf2d6e400a4cc0f7f4a058679d986ddf181cf207692cf0714b956a6490c2a0ceb290e938
+  data.tar.gz: 192614cfa849cacf28cdabf63727d648d421c795a4a6043ac77b3d5b4b1048440ad6dd2dc06fb5b02fe667efc0bf9c7f4eb177bde2b7b9414ddf4fb32269718c

data/ext/arrow/converters.cpp

@@ -25,6 +25,11 @@ namespace red_arrow {
     return list_array_value_converter_->convert(array, i);
   }
 
+  VALUE ArrayValueConverter::convert(const arrow::LargeListArray& array,
+                                     const int64_t i) {
+    return large_list_array_value_converter_->convert(array, i);
+  }
+
   VALUE ArrayValueConverter::convert(const arrow::StructArray& array,
                                      const int64_t i) {
     return struct_array_value_converter_->convert(array, i);

data/ext/arrow/converters.hpp

@@ -28,6 +28,7 @@
 
 namespace red_arrow {
   class ListArrayValueConverter;
+  class LargeListArrayValueConverter;
   class StructArrayValueConverter;
   class MapArrayValueConverter;
   class UnionArrayValueConverter;
@@ -38,6 +39,7 @@ namespace red_arrow {
     ArrayValueConverter()
       : decimal_buffer_(),
         list_array_value_converter_(nullptr),
+        large_list_array_value_converter_(nullptr),
         struct_array_value_converter_(nullptr),
         map_array_value_converter_(nullptr),
         union_array_value_converter_(nullptr),
@@ -45,11 +47,13 @@ namespace red_arrow {
     }
 
     inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter,
+                                         LargeListArrayValueConverter* large_list_array_value_converter,
                                          StructArrayValueConverter* struct_array_value_converter,
                                          MapArrayValueConverter* map_array_value_converter,
                                          UnionArrayValueConverter* union_array_value_converter,
                                          DictionaryArrayValueConverter* dictionary_array_value_converter) {
       list_array_value_converter_ = list_array_value_converter;
+      large_list_array_value_converter_ = large_list_array_value_converter;
       struct_array_value_converter_ = struct_array_value_converter;
       map_array_value_converter_ = map_array_value_converter;
       union_array_value_converter_ = union_array_value_converter;
@@ -263,6 +267,9 @@ namespace red_arrow {
     VALUE convert(const arrow::ListArray& array,
                   const int64_t i);
 
+    VALUE convert(const arrow::LargeListArray& array,
+                  const int64_t i);
+
     VALUE convert(const arrow::StructArray& array,
                   const int64_t i);
 
@@ -298,6 +305,7 @@ namespace red_arrow {
 
     std::string decimal_buffer_;
     ListArrayValueConverter* list_array_value_converter_;
+    LargeListArrayValueConverter* large_list_array_value_converter_;
     StructArrayValueConverter* struct_array_value_converter_;
     MapArrayValueConverter* map_array_value_converter_;
     UnionArrayValueConverter* union_array_value_converter_;
@@ -359,6 +367,106 @@ namespace red_arrow {
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
     VISIT(List)
+    VISIT(LargeList)
+    VISIT(Struct)
+    VISIT(Map)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    VISIT(Decimal256)
+    // TODO
+    // VISIT(Extension)
+
+#undef VISIT
+
+  private:
+    template <typename ArrayType>
+    inline VALUE convert_value(const ArrayType& array,
+                               const int64_t i) {
+      return array_value_converter_->convert(array, i);
+    }
+
+    template <typename ArrayType>
+    arrow::Status visit_value(const ArrayType& array) {
+      if (array.null_count() > 0) {
+        for (int64_t i = 0; i < length_; ++i) {
+          auto value = Qnil;
+          if (!array.IsNull(i + offset_)) {
+            value = convert_value(array, i + offset_);
+          }
+          rb_ary_push(result_, value);
+        }
+      } else {
+        for (int64_t i = 0; i < length_; ++i) {
+          rb_ary_push(result_, convert_value(array, i + offset_));
+        }
+      }
+      return arrow::Status::OK();
+    }
+
+    ArrayValueConverter* array_value_converter_;
+    int32_t offset_;
+    int32_t length_;
+    VALUE result_;
+  };
+
+  class LargeListArrayValueConverter : public arrow::ArrayVisitor {
+  public:
+    explicit LargeListArrayValueConverter(ArrayValueConverter* converter)
+      : array_value_converter_(converter),
+        offset_(0),
+        length_(0),
+        result_(Qnil) {}
+
+    VALUE convert(const arrow::LargeListArray& array, const int64_t index) {
+      auto values = array.values().get();
+      auto offset_keep = offset_;
+      auto length_keep = length_;
+      offset_ = array.value_offset(index);
+      length_ = array.value_length(index);
+      auto result_keep = result_;
+      result_ = rb_ary_new_capa(length_);
+      check_status(values->Accept(this),
+                   "[raw-records][large-list-array]");
+      offset_ = offset_keep;
+      length_ = length_keep;
+      auto result_return = result_;
+      result_ = result_keep;
+      return result_return;
+    }
+
+#define VISIT(TYPE)                                                     \
+    arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
+      return visit_value(array);                                        \
+    }
+
+    VISIT(Null)
+    VISIT(Boolean)
+    VISIT(Int8)
+    VISIT(Int16)
+    VISIT(Int32)
+    VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    VISIT(MonthInterval)
+    VISIT(DayTimeInterval)
+    VISIT(MonthDayNanoInterval)
+    VISIT(List)
+    VISIT(LargeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -465,6 +573,7 @@ namespace red_arrow {
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
     VISIT(List)
+    VISIT(LargeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -567,6 +676,7 @@ namespace red_arrow {
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
     VISIT(List)
+    VISIT(LargeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -670,6 +780,7 @@ namespace red_arrow {
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
     VISIT(List)
+    VISIT(LargeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -781,6 +892,7 @@ namespace red_arrow {
     VISIT(DayTimeInterval)
     VISIT(MonthDayNanoInterval)
     VISIT(List)
+    VISIT(LargeList)
     VISIT(Struct)
     VISIT(Map)
     VISIT(SparseUnion)
@@ -810,12 +922,14 @@ namespace red_arrow {
     explicit Converter()
       : array_value_converter_(),
         list_array_value_converter_(&array_value_converter_),
+        large_list_array_value_converter_(&array_value_converter_),
         struct_array_value_converter_(&array_value_converter_),
         map_array_value_converter_(&array_value_converter_),
         union_array_value_converter_(&array_value_converter_),
         dictionary_array_value_converter_(&array_value_converter_) {
       array_value_converter_.
         set_sub_value_converters(&list_array_value_converter_,
+                                 &large_list_array_value_converter_,
                                  &struct_array_value_converter_,
                                  &map_array_value_converter_,
                                  &union_array_value_converter_,
@@ -830,6 +944,7 @@ namespace red_arrow {
 
     ArrayValueConverter array_value_converter_;
     ListArrayValueConverter list_array_value_converter_;
+    LargeListArrayValueConverter large_list_array_value_converter_;
     StructArrayValueConverter struct_array_value_converter_;
     MapArrayValueConverter map_array_value_converter_;
     UnionArrayValueConverter union_array_value_converter_;

data/ext/arrow/values.cpp

@@ -80,6 +80,7 @@ namespace red_arrow {
       VISIT(DayTimeInterval)
       VISIT(MonthDayNanoInterval)
       VISIT(List)
+      VISIT(LargeList)
       VISIT(Struct)
       VISIT(Map)
       VISIT(SparseUnion)

data/lib/arrow/array-builder.rb

@@ -74,14 +74,23 @@ module Arrow
             detected: true,
           }
         when Integer
-          if value < 0
+          builder_info ||= {}
+          min = builder_info[:min] || value
+          max = builder_info[:max] || value
+          min = value if value < min
+          max = value if value > max
+
+          if builder_info[:builder_type] == :int || value < 0
             {
-              builder: IntArrayBuilder.new,
-              detected: true,
+              builder_type: :int,
+              min: min,
+              max: max,
             }
           else
             {
-              builder: UIntArrayBuilder.new,
+              builder_type: :uint,
+              min: min,
+              max: max,
             }
           end
         when Time
@@ -150,17 +159,20 @@ module Arrow
             end
           end
         when ::Array
-          sub_builder_info = nil
+          sub_builder_info = builder_info && builder_info[:value_builder_info]
           value.each do |sub_value|
             sub_builder_info = detect_builder_info(sub_value, sub_builder_info)
             break if sub_builder_info and sub_builder_info[:detected]
           end
-          if sub_builder_info and sub_builder_info[:detected]
-            sub_value_data_type = sub_builder_info[:builder].value_data_type
+          if sub_builder_info
+            sub_builder = sub_builder_info[:builder] || create_builder(sub_builder_info)
+            return sub_builder_info unless sub_builder
+            sub_value_data_type = sub_builder.value_data_type
             field = Field.new("item", sub_value_data_type)
             {
               builder: ListArrayBuilder.new(ListDataType.new(field)),
-              detected: true,
+              value_builder_info: sub_builder_info,
+              detected: sub_builder_info[:detected],
             }
           else
             builder_info
@@ -184,6 +196,35 @@ module Arrow
           data_type = Decimal256DataType.new(builder_info[:precision],
                                              builder_info[:scale])
           Decimal256ArrayBuilder.new(data_type)
+        when :int
+          min = builder_info[:min]
+          max = builder_info[:max]
+
+          if GLib::MININT8 <= min && max <= GLib::MAXINT8
+            Int8ArrayBuilder.new
+          elsif GLib::MININT16 <= min && max <= GLib::MAXINT16
+            Int16ArrayBuilder.new
+          elsif GLib::MININT32 <= min && max <= GLib::MAXINT32
+            Int32ArrayBuilder.new
+          elsif GLib::MININT64 <= min && max <= GLib::MAXINT64
+            Int64ArrayBuilder.new
+          else
+            StringArrayBuilder.new
+          end
+        when :uint
+          max = builder_info[:max]
+
+          if max <= GLib::MAXUINT8
+            UInt8ArrayBuilder.new
+          elsif max <= GLib::MAXUINT16
+            UInt16ArrayBuilder.new
+          elsif max <= GLib::MAXUINT32
+            UInt32ArrayBuilder.new
+          elsif max <= GLib::MAXUINT64
+            UInt64ArrayBuilder.new
+          else
+            StringArrayBuilder.new
+          end
         else
           nil
         end

data/lib/arrow/array-statistics.rb

@@ -17,6 +17,18 @@
 
 module Arrow
   class ArrayStatistics
+    if method_defined?(:null_count_exact)
+      alias_method :null_count_raw, :null_count
+      def null_count
+        return nil unless has_null_count?
+        if null_count_exact?
+          null_count_exact
+        else
+          null_count_approximate
+        end
+      end
+    end
+
     if method_defined?(:distinct_count_exact)
       alias_method :distinct_count_raw, :distinct_count
       def distinct_count

data/lib/arrow/column.rb

@@ -30,6 +30,18 @@ module Arrow
       @container.share_input(@data)
     end
 
+    def to_arrow
+      @data
+    end
+
+    def to_arrow_array
+      @data.to_arrow_array
+    end
+
+    def to_arrow_chunked_array
+      @data.to_arrow_chunked_array
+    end
+
     def name
       @field.name
     end

data/lib/arrow/csv-write-options.rb

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class CSVWriteOptions
+    class << self
+      def try_convert(value)
+        case value
+        when Hash
+          options = new
+          value.each do |k, v|
+            options.public_send("#{k}=", v)
+          end
+          options
+        else
+          nil
+        end
+      end
+    end
+    
+    alias_method :delimiter_raw, :delimiter
+    def delimiter
+      delimiter_raw.chr
+    end
+
+    alias_method :delimiter_raw=, :delimiter=
+    def delimiter=(delimiter)
+      case delimiter
+      when String
+        if delimiter.bytesize != 1
+          message = "delimiter must be 1 byte character: #{delimiter.inspect}"
+          raise ArgumentError, message
+        end
+        delimiter = delimiter.ord
+      end
+      self.delimiter_raw = delimiter
+    end
+  end
+end

data/lib/arrow/duration-array-builder.rb

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DurationArrayBuilder
+    class << self
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+  end
+end

data/lib/arrow/duration-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DurationArray
+    def unit
+      @unit ||= value_data_type.unit
+    end
+  end
+end

data/lib/arrow/duration-data-type.rb

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DurationDataType
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Symbol, Arrow::TimeUnit
+          new(value)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/arrow/fixed-size-list-array-builder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class FixedSizeListArrayBuilder
+    class << self
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+
+    prepend ListValuesAppendable
+  end
+end

data/lib/arrow/large-list-array-builder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class LargeListArrayBuilder
+    class << self
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+
+    prepend ListValuesAppendable
+  end
+end

data/lib/arrow/large-list-data-type.rb

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class LargeListDataType
+    include ListFieldResolvable
+
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    # Creates a new {Arrow::LargeListDataType}.
+    #
+    # @overload initialize(field)
+    #
+    #   @param field [Arrow::Field, Hash] The field of the large list
+    #     data type. You can also specify field description by `Hash`.
+    #
+    #     See {Arrow::Field.new} how to specify field description.
+    #
+    #   @example Create a large list data type with {Arrow::Field}
+    #     visible_field = Arrow::Field.new("visible", :boolean)
+    #     Arrow::LargeListDataType.new(visible_field)
+    #
+    #   @example Create a large list data type with field description
+    #     Arrow::LargeListDataType.new(name: "visible", type: :boolean)
+    #
+    # @overload initialize(description)
+    #
+    #   @param description [Hash] The description of the large list data
+    #     type. It must have `:field` value.
+    #
+    #   @option description [Arrow::Field, Hash] :field The field of
+    #     the large list data type. You can also specify field
+    #     description by `Hash`.
+    #
+    #     See {Arrow::Field.new} how to specify field description.
+    #
+    #   @example Create a large list data type with {Arrow::Field}
+    #     visible_field = Arrow::Field.new("visible", :boolean)
+    #     Arrow::LargeListDataType.new(field: visible_field)
+    #
+    #   @example Create a large list data type with field description
+    #     Arrow::LargeListDataType.new(field: {name: "visible", type: :boolean})
+    #
+    # @overload initialize(data_type)
+    #
+    #   @param data_type [Arrow::DataType, String, Symbol,
+    #     ::Array<String>, ::Array<Symbol>, Hash] The element data
+    #     type of the large list data type. A field is created with the
+    #     default name `"item"` from the data type automatically.
+    #
+    #     See {Arrow::DataType.resolve} how to specify data type.
+    #
+    #   @example Create a large list data type with {Arrow::DataType}
+    #     Arrow::LargeListDataType.new(Arrow::BooleanDataType.new)
+    #
+    #   @example Create a large list data type with data type name as String
+    #     Arrow::LargeListDataType.new("boolean")
+    #
+    #   @example Create a large list data type with data type name as Symbol
+    #     Arrow::LargeListDataType.new(:boolean)
+    #
+    #   @example Create a large list data type with data type as Array
+    #     Arrow::LargeListDataType.new([:time32, :milli])
+    def initialize(arg)
+      initialize_raw(resolve_field(arg))
+    end
+  end
+end

data/lib/arrow/libraries.rb

@@ -21,6 +21,8 @@ require_relative "field-containable"
 require_relative "generic-filterable"
 require_relative "generic-takeable"
 require_relative "input-referable"
+require_relative "list-field-resolvable"
+require_relative "list-values-appendable"
 require_relative "record-containable"
 require_relative "symbol-values-appendable"
 
@@ -37,6 +39,7 @@ require_relative "column"
 require_relative "compression-type"
 require_relative "csv-loader"
 require_relative "csv-read-options"
+require_relative "csv-write-options"
 require_relative "data-type"
 require_relative "date32-array"
 require_relative "date32-array-builder"
@@ -57,6 +60,9 @@ require_relative "dense-union-array-builder"
 require_relative "dense-union-data-type"
 require_relative "dictionary-array"
 require_relative "dictionary-data-type"
+require_relative "duration-array"
+require_relative "duration-array-builder"
+require_relative "duration-data-type"
 require_relative "equal-options"
 require_relative "expression"
 require_relative "field"
@@ -64,13 +70,18 @@ require_relative "file-output-stream"
 require_relative "file-system"
 require_relative "fixed-size-binary-array"
 require_relative "fixed-size-binary-array-builder"
+require_relative "fixed-size-list-array-builder"
 require_relative "function"
 require_relative "group"
 require_relative "half-float"
 require_relative "half-float-array"
 require_relative "half-float-array-builder"
+require_relative "large-list-array-builder"
+require_relative "large-list-data-type"
 require_relative "list-array-builder"
 require_relative "list-data-type"
+require_relative "list-slice-options"
+require_relative "make-struct-options"
 require_relative "map-array"
 require_relative "map-array-builder"
 require_relative "map-data-type"

data/lib/arrow/list-array-builder.rb

@@ -24,74 +24,6 @@ module Arrow
       end
     end
 
-    alias_method :append_value_raw, :append_value
-
-    # @overload append_value
-    #
-    #   Starts appending a list record. You also need to append list
-    #   value by {#value_builder}.
-    #
-    # @overload append_value(list)
-    #
-    #   Appends a list record including list value.
-    #
-    #   @param value [nil, ::Array] The list value of the record.
-    #
-    #     If this is `nil`, the list record is null.
-    #
-    #     If this is `Array`, it's the list value of the record.
-    #
-    # @since 0.12.0
-    def append_value(*args)
-      n_args = args.size
-
-      case n_args
-      when 0
-        append_value_raw
-      when 1
-        value = args[0]
-        case value
-        when nil
-          append_null
-        when ::Array
-          append_value_raw
-          return if value.empty?
-          @value_builder ||= value_builder
-          @value_builder.append(*value)
-        else
-          message = "list value must be nil or Array: #{value.inspect}"
-          raise ArgumentError, message
-        end
-      else
-        message = "wrong number of arguments (given #{n_args}, expected 0..1)"
-        raise ArgumentError, message
-      end
-    end
-
-    def append_values(lists, is_valids=nil)
-      if is_valids
-        is_valids.each_with_index do |is_valid, i|
-          if is_valid
-            append_value(lists[i])
-          else
-            append_null
-          end
-        end
-      else
-        lists.each do |list|
-          append_value(list)
-        end
-      end
-    end
-
-    # @since 0.12.0
-    def append(*values)
-      if values.empty?
-        # For backward compatibility
-        append_value
-      else
-        super
-      end
-    end
+    prepend ListValuesAppendable
   end
 end

data/lib/arrow/list-data-type.rb

@@ -17,6 +17,8 @@
 
 module Arrow
   class ListDataType
+    include ListFieldResolvable
+
     alias_method :initialize_raw, :initialize
     private :initialize_raw
 
@@ -75,39 +77,7 @@ module Arrow
     #   @example Create a list data type with data type as Array
     #     Arrow::ListDataType.new([:time32, :milli])
     def initialize(arg)
-      data_type = resolve_data_type(arg)
-      if data_type
-        field = Field.new(default_field_name, data_type)
-      else
-        field = resolve_field(arg)
-      end
-      initialize_raw(field)
-    end
-
-    private
-    def resolve_data_type(arg)
-      case arg
-      when DataType, String, Symbol, ::Array
-        DataType.resolve(arg)
-      when Hash
-        return nil if arg[:name]
-        return nil unless arg[:type]
-        DataType.resolve(arg)
-      else
-        nil
-      end
-    end
-
-    def default_field_name
-      "item"
-    end
-
-    def resolve_field(arg)
-      if arg.is_a?(Hash) and arg.key?(:field)
-        description = arg
-        arg = description[:field]
-      end
-      arg
+      initialize_raw(resolve_field(arg))
     end
   end
 end

data/lib/arrow/list-field-resolvable.rb

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module ListFieldResolvable
+    private
+    def resolve_data_type(arg)
+      case arg
+      when DataType, String, Symbol, ::Array
+        DataType.resolve(arg)
+      when Hash
+        return nil if arg[:name]
+        return nil unless arg[:type]
+        DataType.resolve(arg)
+      else
+        nil
+      end
+    end
+
+    def default_field_name
+      "item"
+    end
+
+    def resolve_field(arg)
+      data_type = resolve_data_type(arg)
+      if data_type
+        Field.new(default_field_name, data_type)
+      elsif arg.is_a?(Hash) and arg.key?(:field)
+        description = arg
+        description[:field]
+      else
+        arg
+      end
+    end
+  end
+end

data/lib/arrow/list-slice-options.rb

@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class ListSliceOptions
+    alias_method :return_fixed_size_list_raw, :return_fixed_size_list
+    private :return_fixed_size_list_raw
+
+    RETURN_FIXED_SIZE_GLIB_TO_RUBY = {
+      ListSliceReturnFixedSizeList::AUTO => nil,
+      ListSliceReturnFixedSizeList::TRUE => true,
+      ListSliceReturnFixedSizeList::FALSE => false,
+    }.freeze
+
+    RETURN_FIXED_SIZE_RUBY_TO_GLIB = RETURN_FIXED_SIZE_GLIB_TO_RUBY.invert.freeze
+
+    # Whether to return a FixedSizeListArray. If true _and_ stop is after a
+    # list element’s length, nil values will be appended to create the requested
+    # slice size. The default of nil will return the same type which was passed in.
+    #
+    # @since 23.0.0
+    def return_fixed_size_list
+      RETURN_FIXED_SIZE_GLIB_TO_RUBY.fetch(
+        return_fixed_size_list_raw,
+        return_fixed_size_list_raw)
+    end
+
+    alias_method :return_fixed_size_list_raw=, :return_fixed_size_list=
+    private :return_fixed_size_list_raw=
+
+    # Whether to return a FixedSizeListArray. If true _and_ stop is after a
+    # list element’s length, nil values will be appended to create the requested
+    # slice size. The default of nil will return the same type which was passed in.
+    #
+    # @since 23.0.0
+    def return_fixed_size_list=(return_fixed_size_list)
+      self.return_fixed_size_list_raw = RETURN_FIXED_SIZE_RUBY_TO_GLIB.fetch(
+        return_fixed_size_list,
+        return_fixed_size_list)
+    end
+
+    alias_method :stop_raw, :stop
+    private :stop_raw
+
+    # Optional stop of list slicing. If set to nil, then slice to end.
+    #
+    # @since 23.0.0
+    def stop
+      stop_raw == LIST_SLICE_OPTIONS_STOP_UNSPECIFIED ? nil : stop_raw
+    end
+
+    alias_method :stop_raw=, :stop=
+    private :stop_raw=
+
+    # Optional stop of list slicing. If set to nil, then slice to end.
+    #
+    # @since 23.0.0
+    def stop=(stop)
+      self.stop_raw = stop.nil? ? LIST_SLICE_OPTIONS_STOP_UNSPECIFIED : stop
+    end
+  end
+end

data/lib/arrow/list-values-appendable.rb

@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module ListValuesAppendable
+    # @overload append_value
+    #
+    #   Starts appending a list record. You also need to append list
+    #   value by {#value_builder}.
+    #
+    # @overload append_value(list)
+    #
+    #   Appends a list record including list value.
+    #
+    #   @param value [nil, ::Array] The list value of the record.
+    #
+    #     If this is `nil`, the list record is null.
+    #
+    #     If this is `Array`, it's the list value of the record.
+    #
+    # @since 0.12.0
+    def append_value(*args)
+      n_args = args.size
+
+      case n_args
+      when 0
+        super()
+      when 1
+        value = args[0]
+        case value
+        when nil
+          append_null
+        when ::Array
+          super()
+          return if value.empty?
+          @value_builder ||= value_builder
+          @value_builder.append(*value)
+        else
+          message = "list value must be nil or Array: #{value.inspect}"
+          raise ArgumentError, message
+        end
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 0..1)"
+        raise ArgumentError, message
+      end
+    end
+
+    def append_values(lists, is_valids=nil)
+      if is_valids
+        is_valids.each_with_index do |is_valid, i|
+          if is_valid
+            append_value(lists[i])
+          else
+            append_null
+          end
+        end
+      else
+        lists.each do |list|
+          append_value(list)
+        end
+      end
+    end
+
+    # @since 0.12.0
+    def append(*values)
+      if values.empty?
+        # For backward compatibility
+        append_value
+      else
+        super
+      end
+    end
+  end
+end

data/lib/arrow/make-struct-options.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class MakeStructOptions
+    class << self
+      def try_convert(value)
+        case value
+        when Hash
+          options = new
+          field_names = value[:field_names] || []
+          field_nullability = value[:field_nullability] || []
+          field_metadata = value[:field_metadata] || []
+          field_names.zip(field_nullability, field_metadata) do |name, nullability, metadata|
+            options.add_field(name, nullability, metadata)
+          end
+          options
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "22.0.0"
+  VERSION = "23.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -33,13 +33,13 @@ Gem::Specification.new do |spec|
   ]
   spec.version = version_components.compact.join(".")
   spec.homepage = "https://arrow.apache.org/"
-  spec.authors = ["Apache Arrow Developers"]
+  spec.authors = ["The Apache Software Foundation"]
   spec.email = ["dev@arrow.apache.org"]
 
   spec.summary = "Red Arrow is the Ruby bindings of Apache Arrow"
   spec.description =
     "Apache Arrow is a common in-memory columnar data store. " +
-    "It's useful to share and process large data."
+    "It's useful to share and process large data efficiently."
   spec.license = "Apache-2.0"
   spec.files = ["README.md", "Rakefile", "Gemfile", "#{spec.name}.gemspec"]
   spec.files += ["LICENSE.txt", "NOTICE.txt"]

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 22.0.0
+  version: 23.0.0
 platform: ruby
 authors:
-- Apache Arrow Developers
+- The Apache Software Foundation
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
@@ -80,7 +80,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Apache Arrow is a common in-memory columnar data store. It's useful to
-  share and process large data.
+  share and process large data efficiently.
 email:
 - dev@arrow.apache.org
 executables: []
@@ -122,6 +122,7 @@ files:
 - lib/arrow/constructor-arguments-gc-guardable.rb
 - lib/arrow/csv-loader.rb
 - lib/arrow/csv-read-options.rb
+- lib/arrow/csv-write-options.rb
 - lib/arrow/data-type.rb
 - lib/arrow/date32-array-builder.rb
 - lib/arrow/date32-array.rb
@@ -142,6 +143,9 @@ files:
 - lib/arrow/dense-union-data-type.rb
 - lib/arrow/dictionary-array.rb
 - lib/arrow/dictionary-data-type.rb
+- lib/arrow/duration-array-builder.rb
+- lib/arrow/duration-array.rb
+- lib/arrow/duration-data-type.rb
 - lib/arrow/equal-options.rb
 - lib/arrow/expression.rb
 - lib/arrow/field-containable.rb
@@ -150,6 +154,7 @@ files:
 - lib/arrow/file-system.rb
 - lib/arrow/fixed-size-binary-array-builder.rb
 - lib/arrow/fixed-size-binary-array.rb
+- lib/arrow/fixed-size-list-array-builder.rb
 - lib/arrow/function.rb
 - lib/arrow/generic-filterable.rb
 - lib/arrow/generic-takeable.rb
@@ -177,10 +182,16 @@ files:
 - lib/arrow/jruby/stream-listener-raw.rb
 - lib/arrow/jruby/table.rb
 - lib/arrow/jruby/writable.rb
+- lib/arrow/large-list-array-builder.rb
+- lib/arrow/large-list-data-type.rb
 - lib/arrow/libraries.rb
 - lib/arrow/list-array-builder.rb
 - lib/arrow/list-data-type.rb
+- lib/arrow/list-field-resolvable.rb
+- lib/arrow/list-slice-options.rb
+- lib/arrow/list-values-appendable.rb
 - lib/arrow/loader.rb
+- lib/arrow/make-struct-options.rb
 - lib/arrow/map-array-builder.rb
 - lib/arrow/map-array.rb
 - lib/arrow/map-data-type.rb
@@ -246,7 +257,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata:
-  msys2_mingw_dependencies: arrow>=22.0.0
+  msys2_mingw_dependencies: arrow>=23.0.0
 rdoc_options: []
 require_paths:
 - lib
@@ -261,21 +272,21 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- 'system: arrow-glib>=22.0.0: amazon_linux: arrow-glib-devel'
-- 'system: arrow-glib>=22.0.0: amazon_linux: https://packages.apache.org/artifactory/arrow/amazon-linux/%{version}/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=22.0.0: amazon_linux: arrow-glib-devel'
-- 'system: arrow-glib>=22.0.0: centos: arrow-glib-devel'
-- 'system: arrow-glib>=22.0.0: centos: https://packages.apache.org/artifactory/arrow/centos/%{major_version}-stream/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=22.0.0: centos: arrow-glib-devel'
-- 'system: arrow-glib>=22.0.0: conda: arrow-c-glib'
-- 'system: arrow-glib>=22.0.0: debian: libarrow-glib-dev'
-- 'system: arrow-glib>=22.0.0: debian: https://packages.apache.org/artifactory/arrow/%{distribution}/apache-arrow-apt-source-latest-%{code_name}.deb'
-- 'system: arrow-glib>=22.0.0: debian: libarrow-glib-dev'
-- 'system: arrow-glib>=22.0.0: fedora: libarrow-glib-devel'
-- 'system: arrow-glib>=22.0.0: rhel: arrow-glib-devel'
-- 'system: arrow-glib>=22.0.0: rhel: https://packages.apache.org/artifactory/arrow/almalinux/%{major_version}/apache-arrow-release-latest.rpm'
-- 'system: arrow-glib>=22.0.0: rhel: arrow-glib-devel'
-rubygems_version: 4.0.0.dev
+- 'system: arrow-glib>=23.0.0: amazon_linux: arrow-glib-devel'
+- 'system: arrow-glib>=23.0.0: amazon_linux: https://packages.apache.org/artifactory/arrow/amazon-linux/%{version}/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=23.0.0: amazon_linux: arrow-glib-devel'
+- 'system: arrow-glib>=23.0.0: centos: arrow-glib-devel'
+- 'system: arrow-glib>=23.0.0: centos: https://packages.apache.org/artifactory/arrow/centos/%{major_version}-stream/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=23.0.0: centos: arrow-glib-devel'
+- 'system: arrow-glib>=23.0.0: conda: arrow-c-glib'
+- 'system: arrow-glib>=23.0.0: debian: libarrow-glib-dev'
+- 'system: arrow-glib>=23.0.0: debian: https://packages.apache.org/artifactory/arrow/%{distribution}/apache-arrow-apt-source-latest-%{code_name}.deb'
+- 'system: arrow-glib>=23.0.0: debian: libarrow-glib-dev'
+- 'system: arrow-glib>=23.0.0: fedora: libarrow-glib-devel'
+- 'system: arrow-glib>=23.0.0: rhel: arrow-glib-devel'
+- 'system: arrow-glib>=23.0.0: rhel: https://packages.apache.org/artifactory/arrow/almalinux/%{major_version}/apache-arrow-release-latest.rpm'
+- 'system: arrow-glib>=23.0.0: rhel: arrow-glib-devel'
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Red Arrow is the Ruby bindings of Apache Arrow
 test_files: []