red-arrow 9.0.0 → 10.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23d181c0c2d49f1479ddae27c02427d75b72f66a31c097e1f0ead363422170ed
-  data.tar.gz: e2c5e259b177bb136d7f492e46ec3411b5f05d190618cec6736494315fc64efb
+  metadata.gz: dc23de72fe60f7f6b71d45d4ec9019b0066da0de1aaef7992af984fa3ee14db3
+  data.tar.gz: 5383f405645f394c8af4b35ce537a82a54b207823a1debf755d09ef645bcbe15
 SHA512:
-  metadata.gz: a4d7673b6457b4f288d9fc1e0bcc8b9adad239f21dbe722cbb412e81e51d117e8661027731bd4918be19c9ed6c6978f21613ebdd31fbe21271d2175f3d416655
-  data.tar.gz: f300245a89afeb3c8d5e56db26fe866ae666a429429c26155591b430bc47f9d7abe987e35fd0507133b9e5011f78381b6e46797b1072556786fb8fb54e19e9fe
+  metadata.gz: cf9d0c9274059ab09a91af39573c5e4b1a902ca486038e61aff882441d725592bc4143821ba82607979ff09ea7cc9e5220efda1f12478e28b2cf9a9c89cff2d2
+  data.tar.gz: 4225c79d4eeb6770175624ef0eb4e90e263715a1d0b817bf34898ad179a85325f0d15e8fca6697d94870b6147bb965ab1644170fdae52eab07c37396214e5549

data/ext/arrow/extconf.rb

@@ -38,6 +38,8 @@ checking_for(checking_message("Homebrew")) do
   end
 end
 
+$CXXFLAGS += " -std=c++17 "
+
 unless required_pkg_config_package([
                                      "arrow",
                                      Arrow::Version::MAJOR,

data/lib/arrow/array.rb

@@ -47,7 +47,15 @@ module Arrow
             nil
           end
         else
-          nil
+          if value.respond_to?(:to_arrow_array)
+            begin
+              value.to_arrow_array
+            rescue RangeError
+              nil
+            end
+          else
+            nil
+          end
         end
       end
     end
@@ -103,6 +111,10 @@ module Arrow
       self
     end
 
+    def to_arrow_array
+      self
+    end
+
     alias_method :value_data_type_raw, :value_data_type
     def value_data_type
       @value_data_type ||= value_data_type_raw

data/lib/arrow/data-type.rb

@@ -188,9 +188,13 @@ module Arrow
       end
     end
 
-    def build_array(values)
+    def array_class
       base_name = self.class.name.gsub(/DataType\z/, "")
-      builder_class = self.class.const_get("#{base_name}ArrayBuilder")
+      ::Arrow.const_get("#{base_name}Array")
+    end
+
+    def build_array(values)
+      builder_class = array_class.builder_class
       args = [values]
       args.unshift(self) unless builder_class.buildable?(args)
       builder_class.build(*args)

data/lib/arrow/raw-table-converter.rb

@@ -35,7 +35,12 @@ module Arrow
         fields = []
         @values = []
         @raw_table.each do |name, array|
-          array = ArrayBuilder.build(array) if array.is_a?(::Array)
+          if array.respond_to?(:to_arrow_array)
+            array = array.to_arrow_array
+          else
+            array = array.to_ary if array.respond_to?(:to_ary)
+            array = ArrayBuilder.build(array)
+          end
           fields << Field.new(name.to_s, array.value_data_type)
           @values << array
         end

data/lib/arrow/raw-tensor-converter.rb

@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class RawTensorConverter
+    attr_reader :data_type
+    attr_reader :data
+    attr_reader :shape
+    attr_reader :strides
+    attr_reader :dimension_names
+    def initialize(raw_tensor,
+                   data_type: nil,
+                   shape: nil,
+                   strides: nil,
+                   dimension_names: nil)
+      @raw_tensor = raw_tensor
+      @data_type = data_type
+      @data = nil
+      @shape = shape
+      @strides = strides
+      @dimension_names = dimension_names
+      convert
+    end
+
+    private
+    def convert
+      case @raw_tensor
+      when Buffer
+        @data = @raw_tensor
+      when String
+        unless @raw_tensor.encoding == Encoding::ASCII_8BIT
+          message = "raw tensor String must be an ASCII-8BIT encoded string: " +
+                    "#{@raw_tensor.encoding.inspect}"
+          raise ArgumentError, message
+        end
+        @data = Arrow::Buffer.new(@raw_tensor)
+      else
+        @shape ||= guess_shape
+        build_buffer
+        unless @strides.nil?
+          message = "strides: is only accepted with " +
+                    "an Arrow::Buffer or String raw tensor: #{@strides.inspect}"
+          raise ArgumentError, message
+        end
+      end
+      if @shape.nil?
+        raise ArgumentError, "shape: is missing: #{@raw_tensor.inspect}"
+      end
+      if @data_type.nil?
+        raise ArgumentError, "data_type: is missing: #{@raw_tensor.inspect}"
+      end
+    end
+
+    def guess_shape
+      shape = [@raw_tensor.size]
+      target = @raw_tensor[0]
+      while target.is_a?(::Array)
+        shape << target.size
+        target = target[0]
+      end
+      shape
+    end
+
+    def build_buffer
+      if @data_type
+        @data_type = DataType.resolve(@data_type)
+        array = @data_type.build_array(@raw_tensor.flatten)
+      else
+        array = Array.new(@raw_tensor.flatten)
+        @data_type = array.value_data_type
+      end
+      @data = array.data_buffer
+    end
+  end
+end

data/lib/arrow/tensor.rb

@@ -15,10 +15,150 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require_relative "raw-tensor-converter"
+
 module Arrow
   class Tensor
+    alias_method :initialize_raw, :initialize
+    # Creates a new {Arrow::Tensor}.
+    #
+    # @overload initialize(raw_tensor, data_type: nil, shape: nil, dimension_names: nil)
+    #
+    #   @param raw_tensor [::Array<Numeric>] The tensor represented as a
+    #     raw `Array` (not `Arrow::Array`) and `Numeric`s. You can
+    #     pass a nested `Array` for a multi-dimensional tensor.
+    #
+    #   @param data_type [Arrow::DataType, String, Symbol, ::Array<String>,
+    #     ::Array<Symbol>, Hash, nil] The element data type of the tensor.
+    #
+    #     If you specify `nil`, data type is guessed from `raw_tensor`.
+    #
+    #     See {Arrow::DataType.resolve} for how to specify data type.
+    #
+    #   @param shape [::Array<Integer>, nil] The array of dimension sizes.
+    #
+    #     If you specify `nil`, shape is guessed from `raw_tensor`.
+    #
+    #   @param dimension_names [::Array<String>, ::Array<Symbol>, nil]
+    #     The array of the dimension names.
+    #
+    #     If you specify `nil`, all dimensions have empty names.
+    #
+    #   @example Create a tensor from Ruby's Array
+    #     raw_tensor = [
+    #       [
+    #         [1, 2, 3, 4],
+    #         [5, 6, 7, 8],
+    #       ],
+    #       [
+    #         [9, 10, 11, 12],
+    #         [13, 14, 15, 16],
+    #       ],
+    #       [
+    #         [17, 18, 19, 20],
+    #         [21, 22, 23, 24],
+    #       ],
+    #     ]
+    #     Arrow::Tensor.new(raw_tensor)
+    #
+    #   @since 10.0.0
+    #
+    # @overload initialize(data_type, data, shape, strides, dimension_names)
+    #
+    #   @param data_type [Arrow::DataType, String, Symbol, ::Array<String>,
+    #     ::Array<Symbol>, Hash] The element data type of the tensor.
+    #
+    #     See {Arrow::DataType.resolve} how to specify data type.
+    #
+    #   @param data [Arrow::Buffer, String] The data of the tensor.
+    #
+    #   @param shape [::Array<Integer>] The array of dimension sizes.
+    #
+    #   @param strides [::Array<Integer>, nil] The array of strides which
+    #     is the number of bytes between two adjacent elements in each
+    #     dimension.
+    #
+    #     If you specify `nil` or an empty `Array`, strides are
+    #     guessed from `data_type` and `data`.
+    #
+    #   @param dimension_names [::Array<String>, ::Array<Symbol>, nil]
+    #     The array of the dimension names.
+    #
+    #     If you specify `nil`, all dimensions doesn't have their names.
+    #
+    #   @example Create a table from Arrow::Buffer
+    #     raw_data = [
+    #       1, 2,
+    #       3, 4,
+    #
+    #       5, 6,
+    #       7, 8,
+    #
+    #       9, 10,
+    #       11, 12,
+    #     ]
+    #     data = Arrow::Buffer.new(raw_data.pack("c*").freeze)
+    #     shape = [3, 2, 2]
+    #     strides = []
+    #     names = ["a", "b", "c"]
+    #     Arrow::Tensor.new(:int8, data, shape, strides, names)
+    def initialize(*args,
+                   data_type: nil,
+                   data: nil,
+                   shape: nil,
+                   strides: nil,
+                   dimension_names: nil)
+      n_args = args.size
+      case n_args
+      when 1
+        converter = RawTensorConverter.new(args[0],
+                                           data_type: data_type,
+                                           shape: shape,
+                                           strides: strides,
+                                           dimension_names: dimension_names)
+        data_type = converter.data_type
+        data = converter.data
+        shape = converter.shape
+        strides = converter.strides
+        dimension_names = converter.dimension_names
+      when 0, 2..5
+        data_type = args[0] || data_type
+        data = args[1] || data
+        shape = args[2] || shape
+        strides = args[3] || strides
+        dimension_names = args[4] || dimension_names
+        if data_type.nil?
+          raise ArgumentError, "data_type: is missing: #{data.inspect}"
+        end
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 0..5)"
+        raise ArgumentError, message
+      end
+      initialize_raw(DataType.resolve(data_type),
+                     data,
+                     shape,
+                     strides,
+                     dimension_names)
+    end
+
+    def dimension_names
+      n_dimensions.times.collect do |i|
+        get_dimension_name(i)
+      end
+    end
+
     def to_arrow
       self
     end
+
+    def to_arrow_array
+      if n_dimensions != 1
+        raise RangeError, "must be 1 dimensional tensor: #{shape.inspect}"
+      end
+      value_data_type.array_class.new(size,
+                                      buffer,
+                                      nil,
+                                      0)
+    end
   end
 end

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "9.0.0"
+  VERSION = "10.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/test/test-orc.rb

@@ -164,8 +164,8 @@ class ORCTest < Test::Unit::TestCase
         ]
       end
       assert_equal([
-                     ["boolean1: bool", [pp_values([false, true])]],
-                     ["short1: int16", [pp_values([1024, 2048])]],
+                     ["byte1: int8", [pp_values([1, 100])]],
+                     ["int1: int32", [pp_values([65536, 65536])]],
                    ],
                    dump)
     end

data/test/test-table.rb

@@ -40,6 +40,22 @@ class TableTest < Test::Unit::TestCase
     @table = Arrow::Table.new(schema, [@count_array, @visible_array])
   end
 
+  sub_test_case(".new") do
+    test("{Symbol: Arrow::Tensor}") do
+      assert_equal(Arrow::Table.new(numbers: Arrow::UInt8Array.new([1, 2, 3])),
+                   Arrow::Table.new(numbers: Arrow::Tensor.new([1, 2, 3])))
+    end
+
+    test("{Symbol: #to_ary}") do
+      array_like = Object.new
+      def array_like.to_ary
+        [1, 2, 3]
+      end
+      assert_equal(Arrow::Table.new(numbers: Arrow::UInt8Array.new([1, 2, 3])),
+                   Arrow::Table.new(numbers: array_like))
+    end
+  end
+
   test("#columns") do
     assert_equal([
                    Arrow::Column.new(@table, 0),

data/test/test-tensor.rb

@@ -16,6 +16,233 @@
 # under the License.
 
 class TensorTest < Test::Unit::TestCase
+  sub_test_case("class methods") do
+    sub_test_case(".new") do
+      def setup
+        @raw_tensor = [
+          [
+            [1, 2, 3, 4],
+            [5, 6, 7, 8],
+          ],
+          [
+            [9, 10, 11, 12],
+            [13, 14, 15, 16],
+          ],
+          [
+            [17, 18, 19, 20],
+            [21, 22, 23, 24],
+          ],
+        ]
+        @shape = [3, 2, 4]
+        @strides = [8, 4, 1]
+      end
+
+      test("Array") do
+        tensor = Arrow::Tensor.new(@raw_tensor)
+        assert_equal({
+                       value_data_type: Arrow::UInt8DataType.new,
+                       buffer: @raw_tensor.flatten.pack("C*"),
+                       shape: @shape,
+                       strides: @strides,
+                       dimension_names: ["", "", ""],
+                     },
+                     {
+                       value_data_type: tensor.value_data_type,
+                       buffer: tensor.buffer.data.to_s,
+                       shape: tensor.shape,
+                       strides: tensor.strides,
+                       dimension_names: tensor.dimension_names,
+                     })
+      end
+
+      test("Array, data_type: Symbol") do
+        tensor = Arrow::Tensor.new(@raw_tensor, data_type: :int32)
+        assert_equal({
+                       value_data_type: Arrow::Int32DataType.new,
+                       buffer: @raw_tensor.flatten.pack("l*"),
+                       shape: @shape,
+                       strides: @strides.collect {|x| x * 4},
+                       dimension_names: ["", "", ""],
+                     },
+                     {
+                       value_data_type: tensor.value_data_type,
+                       buffer: tensor.buffer.data.to_s,
+                       shape: tensor.shape,
+                       strides: tensor.strides,
+                       dimension_names: tensor.dimension_names,
+                     })
+      end
+
+      test("Array, dimension_names: Array<String>") do
+        tensor = Arrow::Tensor.new(@raw_tensor,
+                                   dimension_names: ["a", "b", "c"])
+        assert_equal({
+                       value_data_type: Arrow::UInt8DataType.new,
+                       buffer: @raw_tensor.flatten.pack("C*"),
+                       shape: @shape,
+                       strides: @strides,
+                       dimension_names: ["a", "b", "c"],
+                     },
+                     {
+                       value_data_type: tensor.value_data_type,
+                       buffer: tensor.buffer.data.to_s,
+                       shape: tensor.shape,
+                       strides: tensor.strides,
+                       dimension_names: tensor.dimension_names,
+                     })
+      end
+
+      test("Array, dimension_names: Array<Symbol>") do
+        tensor = Arrow::Tensor.new(@raw_tensor,
+                                   dimension_names: [:a, :b, :c])
+        assert_equal({
+                       value_data_type: Arrow::UInt8DataType.new,
+                       buffer: @raw_tensor.flatten.pack("C*"),
+                       shape: @shape,
+                       strides: @strides,
+                       dimension_names: ["a", "b", "c"],
+                     },
+                     {
+                       value_data_type: tensor.value_data_type,
+                       buffer: tensor.buffer.data.to_s,
+                       shape: tensor.shape,
+                       strides: tensor.strides,
+                       dimension_names: tensor.dimension_names,
+                     })
+      end
+
+      test("Array, strides:") do
+        message = "strides: is only accepted with " +
+                  "an Arrow::Buffer or String raw tensor: #{@strides.inspect}"
+        assert_raise(ArgumentError.new(message)) do
+          Arrow::Tensor.new(@raw_tensor, strides: @strides)
+        end
+      end
+
+      test("Arrow::Buffer, data_type:, shape:") do
+        data_type = :uint8
+        data = Arrow::Buffer.new(@raw_tensor.flatten.pack("C*").freeze)
+        tensor = Arrow::Tensor.new(data,
+                                   data_type: data_type,
+                                   shape: @shape)
+        assert_equal({
+                       value_data_type: Arrow::UInt8DataType.new,
+                       buffer: @raw_tensor.flatten.pack("C*"),
+                       shape: @shape,
+                       strides: @strides,
+                       dimension_names: ["", "", ""],
+                     },
+                     {
+                       value_data_type: tensor.value_data_type,
+                       buffer: tensor.buffer.data.to_s,
+                       shape: tensor.shape,
+                       strides: tensor.strides,
+                       dimension_names: tensor.dimension_names,
+                     })
+      end
+
+      test("String, data_type:, shape:") do
+        data_type = :uint8
+        data = @raw_tensor.flatten.pack("C*").freeze
+        tensor = Arrow::Tensor.new(data,
+                                   data_type: data_type,
+                                   shape: @shape)
+        assert_equal({
+                       value_data_type: Arrow::UInt8DataType.new,
+                       buffer: @raw_tensor.flatten.pack("C*"),
+                       shape: @shape,
+                       strides: @strides,
+                       dimension_names: ["", "", ""],
+                     },
+                     {
+                       value_data_type: tensor.value_data_type,
+                       buffer: tensor.buffer.data.to_s,
+                       shape: tensor.shape,
+                       strides: tensor.strides,
+                       dimension_names: tensor.dimension_names,
+                     })
+      end
+
+      test("String, data_type:") do
+        data_type = :uint8
+        data = @raw_tensor.flatten.pack("C*").freeze
+        message = "shape: is missing: #{data.inspect}"
+        assert_raise(ArgumentError.new(message)) do
+          Arrow::Tensor.new(data, data_type: data_type)
+        end
+      end
+
+      test("String, shape:") do
+        data = @raw_tensor.flatten.pack("C*").freeze
+        message = "data_type: is missing: #{data.inspect}"
+        assert_raise(ArgumentError.new(message)) do
+          Arrow::Tensor.new(data, shape: @shape)
+        end
+      end
+
+      test("String - not ASCII-8BIT") do
+        data = "XXX"
+        message = "raw tensor String must be " +
+                  "an ASCII-8BIT encoded string: #{data.encoding.inspect}"
+        assert_raise(ArgumentError.new(message)) do
+          Arrow::Tensor.new("XXX")
+        end
+      end
+
+      test("Symbol, Arrow::Buffer, shape:") do
+        data_type = :uint8
+        data = Arrow::Buffer.new(@raw_tensor.flatten.pack("C*").freeze)
+        tensor = Arrow::Tensor.new(data_type,
+                                   data,
+                                   shape: @shape)
+        assert_equal({
+                       value_data_type: Arrow::UInt8DataType.new,
+                       buffer: @raw_tensor.flatten.pack("C*"),
+                       shape: @shape,
+                       strides: @strides,
+                       dimension_names: ["", "", ""],
+                     },
+                     {
+                       value_data_type: tensor.value_data_type,
+                       buffer: tensor.buffer.data.to_s,
+                       shape: tensor.shape,
+                       strides: tensor.strides,
+                       dimension_names: tensor.dimension_names,
+                     })
+      end
+
+      test("Symbol, String, shape:, strides: - !contiguous and column major") do
+        data_type = :uint8
+        @shape[-1] -= 1 # Ignore the last element in @raw_tensor
+        @strides.reverse
+        tensor = Arrow::Tensor.new(data_type,
+                                   @raw_tensor.flatten.pack("C*"),
+                                   shape: @shape,
+                                   strides: @strides)
+        assert_equal({
+                       value_data_type: Arrow::UInt8DataType.new,
+                       buffer: @raw_tensor.flatten.pack("C*"),
+                       shape: @shape,
+                       strides: @strides,
+                       dimension_names: ["", "", ""],
+                       contiguous: false,
+                       row_major: false,
+                       column_major: false,
+                     },
+                     {
+                       value_data_type: tensor.value_data_type,
+                       buffer: tensor.buffer.data.to_s,
+                       shape: tensor.shape,
+                       strides: tensor.strides,
+                       dimension_names: tensor.dimension_names,
+                       contiguous: tensor.contiguous?,
+                       row_major: tensor.row_major?,
+                       column_major: tensor.column_major?,
+                     })
+      end
+    end
+  end
+
   sub_test_case("instance methods") do
     def setup
       raw_data = [
@@ -28,11 +255,11 @@ class TensorTest < Test::Unit::TestCase
         9, 10,
         11, 12,
       ]
-      data = Arrow::Buffer.new(raw_data.pack("c*"))
+      data = Arrow::Buffer.new(raw_data.pack("c*").freeze)
       shape = [3, 2, 2]
       strides = []
       names = ["a", "b", "c"]
-      @tensor = Arrow::Tensor.new(Arrow::Int8DataType.new,
+      @tensor = Arrow::Tensor.new(:int8,
                                   data,
                                   shape,
                                   strides,
@@ -52,5 +279,19 @@ class TensorTest < Test::Unit::TestCase
         end
       end
     end
+
+    sub_test_case("#to_arrow_array") do
+      test("1 dimension") do
+        assert_equal(Arrow::UInt8Array.new([1, 2, 3]),
+                     Arrow::Tensor.new([1, 2, 3]).to_arrow_array)
+      end
+
+      test("2 dimensions") do
+        message = "must be 1 dimensional tensor: [3, 1]"
+        assert_raise(RangeError.new(message)) do
+          Arrow::Tensor.new([[1], [2], [3]]).to_arrow_array
+        end
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 9.0.0
+  version: 10.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-06 00:00:00.000000000 Z
+date: 2022-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -163,6 +163,7 @@ files:
 - lib/arrow/null-array.rb
 - lib/arrow/path-extension.rb
 - lib/arrow/raw-table-converter.rb
+- lib/arrow/raw-tensor-converter.rb
 - lib/arrow/record-batch-builder.rb
 - lib/arrow/record-batch-file-reader.rb
 - lib/arrow/record-batch-iterator.rb
@@ -307,8 +308,8 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata:
-  msys2_mingw_dependencies: arrow>=9.0.0
-post_install_message: 
+  msys2_mingw_dependencies: arrow>=10.0.0
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -324,7 +325,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.3.15
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Red Arrow is the Ruby bindings of Apache Arrow
 test_files: