red-arrow 4.0.1 → 5.0.0

data/test/test-memory-view.rb

@@ -0,0 +1,434 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class MemoryViewTest < Test::Unit::TestCase
+  def setup
+    unless Fiddle.const_defined?(:MemoryView)
+      omit("Fiddle::MemoryView is needed")
+    end
+    unless Fiddle::MemoryView.respond_to?(:export)
+      omit("Fiddle::MemoryView.export is needed")
+    end
+  end
+
+  def little_endian?
+    [1].pack("s") == [1].pack("s<")
+  end
+
+  test("BooleanArray") do
+    array = Arrow::BooleanArray.new([true] * 9)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      if little_endian?
+        template = "b"
+      else
+        template = "B"
+      end
+      assert_equal([
+                     "#{template}8",
+                     1,
+                     2,
+                     [(("1" * 9) + ("0" * 7))].pack("#{template}*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Int8Array") do
+    values = [-(2 ** 7), 0, (2 ** 7) - 1]
+    array = Arrow::Int8Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "c",
+                     1,
+                     values.size,
+                     values.pack("c*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Int16Array") do
+    values = [-(2 ** 15), 0, (2 ** 15) - 1]
+    array = Arrow::Int16Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "s",
+                     2,
+                     2 * values.size,
+                     values.pack("s*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Int32Array") do
+    values = [-(2 ** 31), 0, (2 ** 31) - 1]
+    array = Arrow::Int32Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "l",
+                     4,
+                     4 * values.size,
+                     values.pack("l*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Int64Array") do
+    values = [-(2 ** 63), 0, (2 ** 63) - 1]
+    array = Arrow::Int64Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "q",
+                     8,
+                     8 * values.size,
+                     values.pack("q*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("UInt8Array") do
+    values = [0, (2 ** 8) - 1]
+    array = Arrow::UInt8Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "C",
+                     1,
+                     values.size,
+                     values.pack("C*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("UInt16Array") do
+    values = [0, (2 ** 16) - 1]
+    array = Arrow::UInt16Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "S",
+                     2,
+                     2 * values.size,
+                     values.pack("S*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("UInt32Array") do
+    values = [0, (2 ** 32) - 1]
+    array = Arrow::UInt32Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "L",
+                     4,
+                     4 * values.size,
+                     values.pack("L*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("UInt64Array") do
+    values = [(2 ** 64) - 1]
+    array = Arrow::UInt64Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "Q",
+                     8,
+                     8 * values.size,
+                     values.pack("Q*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("FloatArray") do
+    values = [-1.1, 0.0, 1.1]
+    array = Arrow::FloatArray.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "f",
+                     4,
+                     4 * values.size,
+                     values.pack("f*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("DoubleArray") do
+    values = [-1.1, 0.0, 1.1]
+    array = Arrow::DoubleArray.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "d",
+                     8,
+                     8 * values.size,
+                     values.pack("d*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("FixedSizeBinaryArray") do
+    values = ["\x01\x02", "\x03\x04", "\x05\x06"]
+    data_type = Arrow::FixedSizeBinaryDataType.new(2)
+    array = Arrow::FixedSizeBinaryArray.new(data_type, values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "C2",
+                     2,
+                     2 * values.size,
+                     values.join("").b,
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Date32Array") do
+    n_days_since_epoch = 17406 # 2017-08-28
+    values = [n_days_since_epoch]
+    array = Arrow::Date32Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "l",
+                     4,
+                     4 * values.size,
+                     values.pack("l*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Date64Array") do
+    n_msecs_since_epoch = 1503878400000 # 2017-08-28T00:00:00Z
+    values = [n_msecs_since_epoch]
+    array = Arrow::Date64Array.new(values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "q",
+                     8,
+                     8 * values.size,
+                     values.pack("q*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Time32Array") do
+    values = [1, 2, 3]
+    array = Arrow::Time32Array.new(:milli, values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "l",
+                     4,
+                     4 * values.size,
+                     values.pack("l*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Time64Array") do
+    values = [1, 2, 3]
+    array = Arrow::Time64Array.new(:nano, values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "q",
+                     8,
+                     8 * values.size,
+                     values.pack("q*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("TimestampArray") do
+    values = [1, 2, 3]
+    array = Arrow::TimestampArray.new(:micro, values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "q",
+                     8,
+                     8 * values.size,
+                     values.pack("q*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Decimal128Array") do
+    values = [
+      Arrow::Decimal128.new("10.1"),
+      Arrow::Decimal128.new("11.1"),
+      Arrow::Decimal128.new("10.2"),
+    ]
+    data_type = Arrow::Decimal128DataType.new(3, 1)
+    array = Arrow::Decimal128Array.new(data_type, values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "q2",
+                     16,
+                     16 * values.size,
+                     values.collect {|value| value.to_bytes.to_s}.join(""),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Decimal256Array") do
+    values = [
+      Arrow::Decimal256.new("10.1"),
+      Arrow::Decimal256.new("11.1"),
+      Arrow::Decimal256.new("10.2"),
+    ]
+    data_type = Arrow::Decimal256DataType.new(3, 1)
+    array = Arrow::Decimal256Array.new(data_type, values)
+    Fiddle::MemoryView.export(array) do |memory_view|
+      assert_equal([
+                     "q4",
+                     32,
+                     32 * values.size,
+                     values.collect {|value| value.to_bytes.to_s}.join(""),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+
+  test("Buffer") do
+    values = [0, nil, nil] * 3
+    array = Arrow::Int8Array.new(values)
+    buffer = array.null_bitmap
+    Fiddle::MemoryView.export(buffer) do |memory_view|
+      if little_endian?
+        template = "b"
+      else
+        template = "B"
+      end
+      assert_equal([
+                     "#{template}8",
+                     1,
+                     2,
+                     ["100" * 3].pack("#{template}*"),
+                   ],
+                   [
+                     memory_view.format,
+                     memory_view.item_size,
+                     memory_view.byte_size,
+                     memory_view.to_s,
+                   ])
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 5.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-27 00:00:00.000000000 Z
+date: 2021-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.6
+        version: 3.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.6
+        version: 3.4.5
 - !ruby/object:Gem::Dependency
   name: native-package-installer
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.9
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.9
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -197,6 +211,8 @@ files:
 - ext/arrow/converters.cpp
 - ext/arrow/converters.hpp
 - ext/arrow/extconf.rb
+- ext/arrow/memory-view.cpp
+- ext/arrow/memory-view.hpp
 - ext/arrow/raw-records.cpp
 - ext/arrow/red-arrow.hpp
 - ext/arrow/values.cpp
@@ -211,6 +227,7 @@ files:
 - lib/arrow/column-containable.rb
 - lib/arrow/column.rb
 - lib/arrow/compression-type.rb
+- lib/arrow/constructor-arguments-gc-guardable.rb
 - lib/arrow/csv-loader.rb
 - lib/arrow/csv-read-options.rb
 - lib/arrow/data-type.rb
@@ -218,6 +235,7 @@ files:
 - lib/arrow/date32-array.rb
 - lib/arrow/date64-array-builder.rb
 - lib/arrow/date64-array.rb
+- lib/arrow/datum.rb
 - lib/arrow/decimal128-array-builder.rb
 - lib/arrow/decimal128-array.rb
 - lib/arrow/decimal128-data-type.rb
@@ -229,6 +247,7 @@ files:
 - lib/arrow/dense-union-data-type.rb
 - lib/arrow/dictionary-array.rb
 - lib/arrow/dictionary-data-type.rb
+- lib/arrow/equal-options.rb
 - lib/arrow/field-containable.rb
 - lib/arrow/field.rb
 - lib/arrow/file-output-stream.rb
@@ -252,6 +271,7 @@ files:
 - lib/arrow/record-containable.rb
 - lib/arrow/record.rb
 - lib/arrow/rolling-window.rb
+- lib/arrow/scalar.rb
 - lib/arrow/schema.rb
 - lib/arrow/slicer.rb
 - lib/arrow/sort-key.rb
@@ -303,6 +323,7 @@ files:
 - test/test-array-builder.rb
 - test/test-array.rb
 - test/test-bigdecimal.rb
+- test/test-boolean-scalar.rb
 - test/test-buffer.rb
 - test/test-chunked-array.rb
 - test/test-column.rb
@@ -326,10 +347,13 @@ files:
 - test/test-file-output-stream.rb
 - test/test-fixed-size-binary-array-builder.rb
 - test/test-fixed-size-binary-array.rb
+- test/test-float-scalar.rb
+- test/test-function.rb
 - test/test-group.rb
 - test/test-list-array-builder.rb
 - test/test-list-array.rb
 - test/test-list-data-type.rb
+- test/test-memory-view.rb
 - test/test-null-array.rb
 - test/test-orc.rb
 - test/test-record-batch-builder.rb
@@ -364,8 +388,8 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata:
-  msys2_mingw_dependencies: arrow>=4.0.1
-post_install_message: 
+  msys2_mingw_dependencies: arrow>=5.0.0
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -380,88 +404,92 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Red Arrow is the Ruby bindings of Apache Arrow
 test_files:
-- test/test-decimal256-array-builder.rb
-- test/test-time64-data-type.rb
-- test/test-sort-options.rb
-- test/test-feather.rb
-- test/test-decimal128.rb
-- test/test-struct-array.rb
-- test/test-data-type.rb
-- test/test-list-data-type.rb
-- test/test-dense-union-data-type.rb
-- test/test-decimal256-array.rb
+- test/fixture/TestOrcFile.test1.orc
+- test/fixture/float-integer.csv
+- test/fixture/integer-float.csv
+- test/fixture/null-with-double-quote.csv
+- test/fixture/null-without-double-quote.csv
+- test/fixture/with-header-float.csv
+- test/fixture/with-header.csv
+- test/fixture/without-header-float.csv
+- test/fixture/without-header.csv
+- test/helper/fixture.rb
+- test/helper/omittable.rb
 - test/helper.rb
-- test/test-record-batch.rb
-- test/test-table.rb
+- test/raw-records/test-basic-arrays.rb
+- test/raw-records/test-dense-union-array.rb
+- test/raw-records/test-list-array.rb
+- test/raw-records/test-multiple-columns.rb
+- test/raw-records/test-sparse-union-array.rb
+- test/raw-records/test-struct-array.rb
+- test/raw-records/test-table.rb
 - test/run-test.rb
-- test/test-timestamp-array.rb
-- test/test-date32-array.rb
 - test/test-array-builder.rb
+- test/test-array.rb
+- test/test-bigdecimal.rb
+- test/test-boolean-scalar.rb
+- test/test-buffer.rb
+- test/test-chunked-array.rb
+- test/test-column.rb
+- test/test-csv-loader.rb
+- test/test-data-type.rb
+- test/test-date32-array.rb
 - test/test-date64-array.rb
-- test/test-record-batch-file-reader.rb
+- test/test-decimal128-array-builder.rb
+- test/test-decimal128-array.rb
 - test/test-decimal128-data-type.rb
-- test/test-time.rb
-- test/test-timestamp-data-type.rb
+- test/test-decimal128.rb
+- test/test-decimal256-array-builder.rb
+- test/test-decimal256-array.rb
+- test/test-decimal256-data-type.rb
 - test/test-decimal256.rb
+- test/test-dense-union-data-type.rb
 - test/test-dictionary-array.rb
-- test/test-fixed-size-binary-array-builder.rb
-- test/test-column.rb
+- test/test-dictionary-data-type.rb
+- test/test-feather.rb
 - test/test-field.rb
-- test/test-decimal128-array.rb
-- test/test-csv-loader.rb
-- test/test-bigdecimal.rb
+- test/test-file-output-stream.rb
+- test/test-fixed-size-binary-array-builder.rb
 - test/test-fixed-size-binary-array.rb
-- test/test-list-array.rb
-- test/test-decimal256-data-type.rb
-- test/test-time64-array.rb
-- test/test-rolling-window.rb
-- test/test-dictionary-data-type.rb
-- test/fixture/integer-float.csv
-- test/fixture/without-header-float.csv
-- test/fixture/null-with-double-quote.csv
-- test/fixture/with-header-float.csv
-- test/fixture/TestOrcFile.test1.orc
-- test/fixture/null-without-double-quote.csv
-- test/fixture/without-header.csv
-- test/fixture/with-header.csv
-- test/fixture/float-integer.csv
-- test/test-orc.rb
-- test/test-null-array.rb
-- test/test-time32-data-type.rb
-- test/test-struct-data-type.rb
-- test/test-sort-indices.rb
+- test/test-float-scalar.rb
+- test/test-function.rb
 - test/test-group.rb
-- test/test-sort-key.rb
-- test/test-buffer.rb
-- test/values/test-struct-array.rb
-- test/values/test-basic-arrays.rb
-- test/values/test-list-array.rb
-- test/values/test-dense-union-array.rb
-- test/values/test-sparse-union-array.rb
-- test/test-slicer.rb
 - test/test-list-array-builder.rb
-- test/test-sparse-union-data-type.rb
+- test/test-list-array.rb
+- test/test-list-data-type.rb
+- test/test-memory-view.rb
+- test/test-null-array.rb
+- test/test-orc.rb
 - test/test-record-batch-builder.rb
-- test/raw-records/test-struct-array.rb
-- test/raw-records/test-table.rb
-- test/raw-records/test-basic-arrays.rb
-- test/raw-records/test-list-array.rb
-- test/raw-records/test-dense-union-array.rb
-- test/raw-records/test-sparse-union-array.rb
-- test/raw-records/test-multiple-columns.rb
-- test/test-array.rb
-- test/test-file-output-stream.rb
+- test/test-record-batch-file-reader.rb
 - test/test-record-batch-iterator.rb
-- test/test-tensor.rb
-- test/test-decimal128-array-builder.rb
-- test/test-chunked-array.rb
+- test/test-record-batch.rb
+- test/test-rolling-window.rb
 - test/test-schema.rb
+- test/test-slicer.rb
+- test/test-sort-indices.rb
+- test/test-sort-key.rb
+- test/test-sort-options.rb
+- test/test-sparse-union-data-type.rb
 - test/test-struct-array-builder.rb
-- test/helper/fixture.rb
-- test/helper/omittable.rb
+- test/test-struct-array.rb
+- test/test-struct-data-type.rb
+- test/test-table.rb
+- test/test-tensor.rb
+- test/test-time.rb
 - test/test-time32-array.rb
+- test/test-time32-data-type.rb
+- test/test-time64-array.rb
+- test/test-time64-data-type.rb
+- test/test-timestamp-array.rb
+- test/test-timestamp-data-type.rb
+- test/values/test-basic-arrays.rb
+- test/values/test-dense-union-array.rb
+- test/values/test-list-array.rb
+- test/values/test-sparse-union-array.rb
+- test/values/test-struct-array.rb