red-arrow 8.0.0 → 24.0.0

data/lib/arrow/list-data-type.rb

@@ -17,6 +17,8 @@
 
 module Arrow
   class ListDataType
+    include ListFieldResolvable
+
     alias_method :initialize_raw, :initialize
     private :initialize_raw
 
@@ -75,44 +77,7 @@ module Arrow
     #   @example Create a list data type with data type as Array
     #     Arrow::ListDataType.new([:time32, :milli])
     def initialize(arg)
-      data_type = resolve_data_type(arg)
-      if data_type
-        field = Field.new(default_field_name, data_type)
-      else
-        field = resolve_field(arg)
-      end
-      initialize_raw(field)
-    end
-
-    private
-    def resolve_data_type(arg)
-      case arg
-      when DataType, String, Symbol, ::Array
-        DataType.resolve(arg)
-      when Hash
-        return nil if arg[:name]
-        return nil unless arg[:type]
-        DataType.resolve(arg)
-      else
-        nil
-      end
-    end
-
-    def default_field_name
-      "item"
-    end
-
-    def resolve_field(arg)
-      if arg.is_a?(Hash) and arg.key?(:field)
-        description = arg
-        arg = description[:field]
-      end
-      if arg.is_a?(Hash)
-        field_description = arg
-        Field.new(field_description)
-      else
-        arg
-      end
+      initialize_raw(resolve_field(arg))
     end
   end
 end

data/{test/test-dictionary-array.rb → lib/arrow/list-field-resolvable.rb}

@@ -15,27 +15,36 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class DictionaryArrayTest < Test::Unit::TestCase
-  sub_test_case("instance methods") do
-    def setup
-      @values = ["a", "b", "c", "b", "a"]
-      @string_array = Arrow::StringArray.new(@values)
-      @array = @string_array.dictionary_encode
+module Arrow
+  module ListFieldResolvable
+    private
+    def resolve_data_type(arg)
+      case arg
+      when DataType, String, Symbol, ::Array
+        DataType.resolve(arg)
+      when Hash
+        return nil if arg[:name]
+        return nil unless arg[:type]
+        DataType.resolve(arg)
+      else
+        nil
+      end
     end
 
-    test("#[]") do
-      assert_equal(@values, @array.to_a)
+    def default_field_name
+      "item"
     end
 
-    test("#get_value") do
-      assert_equal([
-                     @values[0],
-                     @values[3],
-                   ],
-                   [
-                     @array.get_value(0),
-                     @array.get_value(3),
-                   ])
+    def resolve_field(arg)
+      data_type = resolve_data_type(arg)
+      if data_type
+        Field.new(default_field_name, data_type)
+      elsif arg.is_a?(Hash) and arg.key?(:field)
+        description = arg
+        description[:field]
+      else
+        arg
+      end
     end
   end
 end

data/lib/arrow/list-slice-options.rb

@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class ListSliceOptions
+    alias_method :return_fixed_size_list_raw, :return_fixed_size_list
+    private :return_fixed_size_list_raw
+
+    RETURN_FIXED_SIZE_GLIB_TO_RUBY = {
+      ListSliceReturnFixedSizeList::AUTO => nil,
+      ListSliceReturnFixedSizeList::TRUE => true,
+      ListSliceReturnFixedSizeList::FALSE => false,
+    }.freeze
+
+    RETURN_FIXED_SIZE_RUBY_TO_GLIB = RETURN_FIXED_SIZE_GLIB_TO_RUBY.invert.freeze
+
+    # Whether to return a FixedSizeListArray. If true _and_ stop is after a
+    # list element’s length, nil values will be appended to create the requested
+    # slice size. The default of nil will return the same type which was passed in.
+    #
+    # @since 23.0.0
+    def return_fixed_size_list
+      RETURN_FIXED_SIZE_GLIB_TO_RUBY.fetch(
+        return_fixed_size_list_raw,
+        return_fixed_size_list_raw)
+    end
+
+    alias_method :return_fixed_size_list_raw=, :return_fixed_size_list=
+    private :return_fixed_size_list_raw=
+
+    # Whether to return a FixedSizeListArray. If true _and_ stop is after a
+    # list element’s length, nil values will be appended to create the requested
+    # slice size. The default of nil will return the same type which was passed in.
+    #
+    # @since 23.0.0
+    def return_fixed_size_list=(return_fixed_size_list)
+      self.return_fixed_size_list_raw = RETURN_FIXED_SIZE_RUBY_TO_GLIB.fetch(
+        return_fixed_size_list,
+        return_fixed_size_list)
+    end
+
+    alias_method :stop_raw, :stop
+    private :stop_raw
+
+    # Optional stop of list slicing. If set to nil, then slice to end.
+    #
+    # @since 23.0.0
+    def stop
+      stop_raw == LIST_SLICE_OPTIONS_STOP_UNSPECIFIED ? nil : stop_raw
+    end
+
+    alias_method :stop_raw=, :stop=
+    private :stop_raw=
+
+    # Optional stop of list slicing. If set to nil, then slice to end.
+    #
+    # @since 23.0.0
+    def stop=(stop)
+      self.stop_raw = stop.nil? ? LIST_SLICE_OPTIONS_STOP_UNSPECIFIED : stop
+    end
+  end
+end

data/lib/arrow/list-values-appendable.rb

@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module ListValuesAppendable
+    # @overload append_value
+    #
+    #   Starts appending a list record. You also need to append list
+    #   value by {#value_builder}.
+    #
+    # @overload append_value(list)
+    #
+    #   Appends a list record including list value.
+    #
+    #   @param value [nil, ::Array] The list value of the record.
+    #
+    #     If this is `nil`, the list record is null.
+    #
+    #     If this is `Array`, it's the list value of the record.
+    #
+    # @since 0.12.0
+    def append_value(*args)
+      n_args = args.size
+
+      case n_args
+      when 0
+        super()
+      when 1
+        value = args[0]
+        case value
+        when nil
+          append_null
+        when ::Array
+          super()
+          return if value.empty?
+          @value_builder ||= value_builder
+          @value_builder.append(*value)
+        else
+          message = "list value must be nil or Array: #{value.inspect}"
+          raise ArgumentError, message
+        end
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 0..1)"
+        raise ArgumentError, message
+      end
+    end
+
+    def append_values(lists, is_valids=nil)
+      if is_valids
+        is_valids.each_with_index do |is_valid, i|
+          if is_valid
+            append_value(lists[i])
+          else
+            append_null
+          end
+        end
+      else
+        lists.each do |list|
+          append_value(list)
+        end
+      end
+    end
+
+    # @since 0.12.0
+    def append(*values)
+      if values.empty?
+        # For backward compatibility
+        append_value
+      else
+        super
+      end
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "arrow/block-closable"
+require_relative "block-closable"
 
 module Arrow
   class Loader < GObjectIntrospection::Loader
@@ -31,103 +31,11 @@ module Arrow
       require_extension_library
       gc_guard
       self.class.start_callback_dispatch_thread
+      @base_module.compute_initialize
     end
 
     def require_libraries
-      require "arrow/column-containable"
-      require "arrow/field-containable"
-      require "arrow/generic-filterable"
-      require "arrow/generic-takeable"
-      require "arrow/record-containable"
-      require "arrow/symbol-values-appendable"
-
-      require "arrow/aggregate-node-options"
-      require "arrow/aggregation"
-      require "arrow/array"
-      require "arrow/array-builder"
-      require "arrow/bigdecimal-extension"
-      require "arrow/binary-dictionary-array-builder"
-      require "arrow/buffer"
-      require "arrow/chunked-array"
-      require "arrow/column"
-      require "arrow/compression-type"
-      require "arrow/csv-loader"
-      require "arrow/csv-read-options"
-      require "arrow/data-type"
-      require "arrow/date32-array"
-      require "arrow/date32-array-builder"
-      require "arrow/date64-array"
-      require "arrow/date64-array-builder"
-      require "arrow/datum"
-      require "arrow/day-time-interval-array-builder"
-      require "arrow/decimal128"
-      require "arrow/decimal128-array"
-      require "arrow/decimal128-array-builder"
-      require "arrow/decimal128-data-type"
-      require "arrow/decimal256"
-      require "arrow/decimal256-array"
-      require "arrow/decimal256-array-builder"
-      require "arrow/decimal256-data-type"
-      require "arrow/dense-union-data-type"
-      require "arrow/dictionary-array"
-      require "arrow/dictionary-data-type"
-      require "arrow/equal-options"
-      require "arrow/expression"
-      require "arrow/field"
-      require "arrow/file-output-stream"
-      require "arrow/file-system"
-      require "arrow/fixed-size-binary-array"
-      require "arrow/fixed-size-binary-array-builder"
-      require "arrow/function"
-      require "arrow/group"
-      require "arrow/list-array-builder"
-      require "arrow/list-data-type"
-      require "arrow/map-array"
-      require "arrow/map-array-builder"
-      require "arrow/map-data-type"
-      require "arrow/month-day-nano-interval-array-builder"
-      require "arrow/null-array"
-      require "arrow/null-array-builder"
-      require "arrow/path-extension"
-      require "arrow/record"
-      require "arrow/record-batch"
-      require "arrow/record-batch-builder"
-      require "arrow/record-batch-file-reader"
-      require "arrow/record-batch-iterator"
-      require "arrow/record-batch-reader"
-      require "arrow/record-batch-stream-reader"
-      require "arrow/rolling-window"
-      require "arrow/s3-global-options"
-      require "arrow/scalar"
-      require "arrow/schema"
-      require "arrow/slicer"
-      require "arrow/sort-key"
-      require "arrow/sort-options"
-      require "arrow/source-node-options"
-      require "arrow/sparse-union-data-type"
-      require "arrow/string-dictionary-array-builder"
-      require "arrow/struct-array"
-      require "arrow/struct-array-builder"
-      require "arrow/struct-data-type"
-      require "arrow/table"
-      require "arrow/table-concatenate-options"
-      require "arrow/table-formatter"
-      require "arrow/table-list-formatter"
-      require "arrow/table-table-formatter"
-      require "arrow/table-loader"
-      require "arrow/table-saver"
-      require "arrow/tensor"
-      require "arrow/time"
-      require "arrow/time32-array"
-      require "arrow/time32-array-builder"
-      require "arrow/time32-data-type"
-      require "arrow/time64-array"
-      require "arrow/time64-array-builder"
-      require "arrow/time64-data-type"
-      require "arrow/timestamp-array"
-      require "arrow/timestamp-array-builder"
-      require "arrow/timestamp-data-type"
-      require "arrow/writable"
+      require_relative "libraries"
     end
 
     def require_extension_library
@@ -135,7 +43,7 @@ module Arrow
     end
 
     def gc_guard
-      require "arrow/constructor-arguments-gc-guardable"
+      require_relative "constructor-arguments-gc-guardable"
 
       [
         @base_module::BinaryScalar,
@@ -155,6 +63,16 @@ module Arrow
       end
     end
 
+    def rubyish_class_name(info)
+      name = info.name
+      case name
+      when "StreamListener"
+        "StreamListenerRaw"
+      else
+        super
+      end
+    end
+
     def load_object_info(info)
       super
 
@@ -193,6 +111,7 @@ module Arrow
            "Arrow::Date64Array",
            "Arrow::Decimal128Array",
            "Arrow::Decimal256Array",
+           "Arrow::HalfFloatArray",
            "Arrow::Time32Array",
            "Arrow::Time64Array",
            "Arrow::TimestampArray"

data/{test/test-decimal128-array.rb → lib/arrow/make-struct-options.rb}

@@ -15,24 +15,24 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class Decimal128ArrayTest < Test::Unit::TestCase
-  sub_test_case(".new") do
-    test("build") do
-      data_type = Arrow::Decimal128DataType.new(3, 1)
-      values = [
-        10.1,
-        nil,
-        "10.1",
-        BigDecimal("10.1"),
-      ]
-      array = Arrow::Decimal128Array.new(data_type, values)
-      assert_equal([
-                     BigDecimal("10.1"),
-                     nil,
-                     BigDecimal("10.1"),
-                     BigDecimal("10.1"),
-                   ],
-                   array.to_a)
+module Arrow
+  class MakeStructOptions
+    class << self
+      def try_convert(value)
+        case value
+        when Hash
+          options = new
+          field_names = value[:field_names] || []
+          field_nullability = value[:field_nullability] || []
+          field_metadata = value[:field_metadata] || []
+          field_names.zip(field_nullability, field_metadata) do |name, nullability, metadata|
+            options.add_field(name, nullability, metadata)
+          end
+          options
+        else
+          nil
+        end
+      end
     end
   end
 end

data/lib/arrow/raw-table-converter.rb

@@ -35,9 +35,16 @@ module Arrow
         fields = []
         @values = []
         @raw_table.each do |name, array|
-          array = ArrayBuilder.build(array) if array.is_a?(::Array)
-          fields << Field.new(name.to_s, array.value_data_type)
-          @values << array
+          if array.respond_to?(:to_arrow_chunked_array)
+            chunked_array = array.to_arrow_chunked_array
+          elsif array.respond_to?(:to_arrow_array)
+            chunked_array = ChunkedArray.new([array.to_arrow_array])
+          else
+            array = array.to_ary if array.respond_to?(:to_ary)
+            chunked_array = ChunkedArray.new([ArrayBuilder.build(array)])
+          end
+          fields << Field.new(name.to_s, chunked_array.value_data_type)
+          @values << chunked_array
         end
         @schema = Schema.new(fields)
       end

data/lib/arrow/raw-tensor-converter.rb

@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class RawTensorConverter
+    attr_reader :data_type
+    attr_reader :data
+    attr_reader :shape
+    attr_reader :strides
+    attr_reader :dimension_names
+    def initialize(raw_tensor,
+                   data_type: nil,
+                   shape: nil,
+                   strides: nil,
+                   dimension_names: nil)
+      @raw_tensor = raw_tensor
+      @data_type = data_type
+      @data = nil
+      @shape = shape
+      @strides = strides
+      @dimension_names = dimension_names
+      convert
+    end
+
+    private
+    def convert
+      case @raw_tensor
+      when Buffer
+        @data = @raw_tensor
+      when String
+        unless @raw_tensor.encoding == Encoding::ASCII_8BIT
+          message = "raw tensor String must be an ASCII-8BIT encoded string: " +
+                    "#{@raw_tensor.encoding.inspect}"
+          raise ArgumentError, message
+        end
+        @data = Arrow::Buffer.new(@raw_tensor)
+      else
+        @shape ||= guess_shape
+        build_buffer
+        unless @strides.nil?
+          message = "strides: is only accepted with " +
+                    "an Arrow::Buffer or String raw tensor: #{@strides.inspect}"
+          raise ArgumentError, message
+        end
+      end
+      if @shape.nil?
+        raise ArgumentError, "shape: is missing: #{@raw_tensor.inspect}"
+      end
+      if @data_type.nil?
+        raise ArgumentError, "data_type: is missing: #{@raw_tensor.inspect}"
+      end
+    end
+
+    def guess_shape
+      shape = [@raw_tensor.size]
+      target = @raw_tensor[0]
+      while target.is_a?(::Array)
+        shape << target.size
+        target = target[0]
+      end
+      shape
+    end
+
+    def build_buffer
+      if @data_type
+        @data_type = DataType.resolve(@data_type)
+        array = @data_type.build_array(@raw_tensor.flatten)
+      else
+        array = Array.new(@raw_tensor.flatten)
+        @data_type = array.value_data_type
+      end
+      @data = array.data_buffer
+    end
+  end
+end

data/lib/arrow/record-batch-file-reader.rb

@@ -20,6 +20,8 @@ module Arrow
     include Enumerable
 
     def each
+      return to_enum(__method__) {n_record_batches} unless block_given?
+
       n_record_batches.times do |i|
         yield(get_record_batch(i))
       end

data/lib/arrow/record-batch-stream-reader.rb

@@ -20,6 +20,8 @@ module Arrow
     include Enumerable
 
     def each
+      return to_enum(__method__) unless block_given?
+
       loop do
         record_batch = next_record_batch
         break if record_batch.nil?

data/lib/arrow/record-batch.rb

@@ -19,9 +19,11 @@ require "arrow/raw-table-converter"
 
 module Arrow
   class RecordBatch
+    include Enumerable
+
     include ColumnContainable
+    include InputReferable
     include RecordContainable
-    include Enumerable
 
     class << self
       def new(*args)
@@ -56,7 +58,9 @@ module Arrow
     #
     # @since 0.12.0
     def to_table
-      Table.new(schema, [self])
+      table = Table.new(schema, [self])
+      share_input(table)
+      table
     end
 
     def respond_to_missing?(name, include_private)

data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb}

@@ -14,7 +14,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-name,score
-alice,10
-bob,
-chris,-1
+
+require "gio2"
+
+require_relative "loader"
+Arrow::Loader.load