red-arrow 0.16.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64b14ef4120f4ab290e8161020902ec2a22631c519d5a133a63ce383610e8545
-  data.tar.gz: 2f5850520e2dc69568a454cee0d4246909d52f1d49851221b1b9efd3149bc15c
+  metadata.gz: ba4a00fdcc27e2e75985b0d0da37eff70530aef9431d5234da72517313ebad55
+  data.tar.gz: 50fb004195b273ec5cbf3e9fa7c92fcb3a7755f010e2ddd469d84c7c27a104df
 SHA512:
-  metadata.gz: 0e19a4da6182437a51f9dad6212436e70b00881674ee6cd29e2a40910fe711fdef4076c81d2778f3ff9e9dd3f45b573c43e90378244cf08d7550b504ad8b53af
-  data.tar.gz: 547eb8b31fd59d9c1d5fc1163bb25da70154b797d103d3a01f6fda70dddc0b3c2cdb5391b9006c4510be37b8b75988195f809aaead8c91e7fb68f762cc5313de
+  metadata.gz: 0d0e852a7d0ea89c03ac752e779a1bd5b0ec6a81a10b6b457d66857f2dacb5e445dc41f5254e8985f3d7b8ec6328cedea92bd22758b84a7e935278374e51fcd3
+  data.tar.gz: 4aa5f3c817f2ef1a475ed7f272f37f495d4f9c66f07a98381a04a6b16ee9b803904f2c0982048003b0bbe6038369d33b3632b6cb437d849d25179b504233e142

data/Rakefile

@@ -30,36 +30,44 @@ spec = helper.gemspec
 release_task = Rake::Task["release"]
 release_task.prerequisites.replace(["build", "release:rubygem_push"])
 
-def run_extconf(extension_dir, *arguments)
-  cd(extension_dir) do
-    ruby("extconf.rb", *arguments)
+def run_extconf(build_dir, extension_dir, *arguments)
+  cd(build_dir) do
+    ruby(File.join(extension_dir, "extconf.rb"),
+         *arguments)
   end
 end
 
 spec.extensions.each do |extension|
-  extension_dir = File.dirname(extension)
-  CLOBBER << File.join(extension_dir, "Makefile")
-  CLOBBER << File.join(extension_dir, "mkmf.log")
+  extension_dir = File.join(base_dir, File.dirname(extension))
+  build_dir = ENV["BUILD_DIR"]
+  if build_dir
+    build_dir = File.join(build_dir, "red-arrow")
+    directory build_dir
+  else
+    build_dir = extension_dir
+  end
+  CLOBBER << File.join(build_dir, "Makefile")
+  CLOBBER << File.join(build_dir, "mkmf.log")
 
-  makefile = File.join(extension_dir, "Makefile")
-  file makefile do
-    run_extconf(extension_dir)
+  makefile = File.join(build_dir, "Makefile")
+  file makefile => build_dir do
+    run_extconf(build_dir, extension_dir)
   end
 
   desc "Configure"
-  task :configure do
-    run_extconf(extension_dir)
+  task :configure => build_dir do
+    run_extconf(build_dir, extension_dir)
   end
 
   desc "Compile"
   task :compile => makefile do
-    cd(extension_dir) do
+    cd(build_dir) do
       sh("make")
     end
   end
 
   task :clean do
-    cd(extension_dir) do
+    cd(build_dir) do
       sh("make", "clean") if File.exist?("Makefile")
     end
   end
@@ -67,7 +75,9 @@ end
 
 desc "Run tests"
 task :test do
-  ruby("test/run-test.rb")
+  cd(base_dir) do
+    ruby("test/run-test.rb")
+  end
 end
 
 task default: :test
@@ -79,8 +89,10 @@ task :benchmark do
                else
                  FileList["benchmark/{,*/**/}*.yml"]
                end
-  benchmarks.each do |benchmark|
-    sh("benchmark-driver", benchmark)
+  cd(base_dir) do
+    benchmarks.each do |benchmark|
+      sh("benchmark-driver", benchmark)
+    end
   end
 end
 

data/lib/arrow/generic-filterable.rb

@@ -19,24 +19,24 @@ module Arrow
   module GenericFilterable
     class << self
       def included(base)
-        base.alias_method :filter_raw, :filter
-        base.alias_method :filter, :filter_generic
+        base.__send__(:alias_method, :filter_raw, :filter)
+        base.__send__(:alias_method, :filter, :filter_generic)
       end
     end
 
-    def filter_generic(filter)
+    def filter_generic(filter, options=nil)
       case filter
       when ::Array
-        filter_raw(BooleanArray.new(filter))
+        filter_raw(BooleanArray.new(filter), options)
       when ChunkedArray
         if respond_to?(:filter_chunked_array)
-          filter_chunked_array(filter)
+          filter_chunked_array(filter, options)
         else
           # TODO: Implement this in C++
-          filter_raw(filter.pack)
+          filter_raw(filter.pack, options)
         end
       else
-        filter_raw(filter)
+        filter_raw(filter, options)
       end
     end
   end

data/lib/arrow/generic-takeable.rb

@@ -19,8 +19,8 @@ module Arrow
   module GenericTakeable
     class << self
       def included(base)
-        base.alias_method :take_raw, :take
-        base.alias_method :take, :take_generic
+        base.__send__(:alias_method, :take_raw, :take)
+        base.__send__(:alias_method, :take, :take_generic)
       end
     end
 

data/lib/arrow/null-array-builder.rb

@@ -19,7 +19,7 @@ module Arrow
   class NullArrayBuilder
     class << self
       def buildable?(args)
-        super and args.collect(&:class) != [Integer]
+        super and not (args.size == 1 and args[0].is_a?(Integer))
       end
     end
   end

data/lib/arrow/schema.rb

@@ -91,5 +91,10 @@ module Arrow
     end
 
     alias_method :[], :find_field
+
+    alias_method :to_s_raw, :to_s
+    def to_s(show_metadata: false)
+      to_string_metadata(show_metadata)
+    end
   end
 end

data/lib/arrow/table-saver.rb

@@ -155,10 +155,14 @@ module Arrow
     end
 
     def save_as_feather
-      open_output_stream do |output|
-        FeatherFileWriter.open(output) do |writer|
-          writer.write(@table)
+      open_raw_output_stream do |output|
+        properties = FeatherWriteProperties.new
+        properties.class.properties.each do |name|
+          value = @options[name.to_sym]
+          next if value.nil?
+          properties.__send__("#{name}=", value)
         end
+        @table.write_as_feather(output, properties)
       end
     end
   end

data/lib/arrow/table.rb

@@ -304,6 +304,8 @@ module Arrow
         end
       end
 
+      filter_options = Arrow::FilterOptions.new
+      filter_options.null_selection_behavior = :emit_null
       sliced_tables = []
       slicers.each do |slicer|
         slicer = slicer.evaluate if slicer.respond_to?(:evaluate)
@@ -325,7 +327,7 @@ module Arrow
           to += n_rows if to < 0
           sliced_tables << slice_by_range(from, to)
         when ::Array, BooleanArray, ChunkedArray
-          sliced_tables << filter(slicer)
+          sliced_tables << filter(slicer, filter_options)
         else
           message = "slicer must be Integer, Range, (from, to), " +
             "Arrow::ChunkedArray of Arrow::BooleanArray, " +

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "0.16.0"
+  VERSION = "0.17.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/test/helper.rb

@@ -24,3 +24,4 @@ require "zlib"
 require "test-unit"
 
 require_relative "helper/fixture"
+require_relative "helper/omittable"

data/test/helper/omittable.rb

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Helper
+  module Omittable
+    def require_gi_bindings(major, minor, micro)
+      return if GLib.check_binding_version?(major, minor, micro)
+      message =
+        "Require gobject-introspection #{major}.#{minor}.#{micro} or later: " +
+        GLib::BINDING_VERSION.join(".")
+      omit(message)
+    end
+
+    def require_gi(major, minor, micro)
+      return if GObjectIntrospection::Version.or_later?(major, minor, micro)
+      message =
+        "Require GObject Introspection #{major}.#{minor}.#{micro} or later: " +
+        GObjectIntrospection::Version::STRING
+      omit(message)
+    end
+  end
+end

data/test/run-test.rb

@@ -19,6 +19,7 @@
 
 $VERBOSE = true
 
+require "fileutils"
 require "pathname"
 
 (ENV["ARROW_DLL_PATH"] || "").split(File::PATH_SEPARATOR).each do |path|
@@ -31,6 +32,14 @@ lib_dir = base_dir + "lib"
 ext_dir = base_dir + "ext" + "arrow"
 test_dir = base_dir + "test"
 
+build_dir = ENV["BUILD_DIR"]
+if build_dir
+  build_dir = File.join(build_dir, "red-arrow")
+  FileUtils.mkdir_p(build_dir)
+else
+  build_dir = ext_dir
+end
+
 make = nil
 if ENV["NO_MAKE"] != "yes"
   if ENV["MAKE"]
@@ -42,15 +51,17 @@ if ENV["NO_MAKE"] != "yes"
   end
 end
 if make
-  Dir.chdir(ext_dir.to_s) do
+  Dir.chdir(build_dir.to_s) do
     unless File.exist?("Makefile")
-      system(RbConfig.ruby, "extconf.rb", "--enable-debug-build") or exit(false)
+      system(RbConfig.ruby,
+             (ext_dir + "extconf.rb").to_s,
+             "--enable-debug-build") or exit(false)
     end
     system("#{make} > #{File::NULL}") or exit(false)
   end
 end
 
-$LOAD_PATH.unshift(ext_dir.to_s)
+$LOAD_PATH.unshift(build_dir.to_s)
 $LOAD_PATH.unshift(lib_dir.to_s)
 
 require_relative "helper"

data/test/test-array.rb

@@ -76,20 +76,22 @@ class ArrayTest < Test::Unit::TestCase
     def setup
       values = [true, false, false, true]
       @array = Arrow::BooleanArray.new(values)
+      @options = Arrow::FilterOptions.new
+      @options.null_selection_behavior = :emit_null
     end
 
     test("Array: boolean") do
       filter = [nil, true, true, false]
       filtered_array = Arrow::BooleanArray.new([nil, false, false])
       assert_equal(filtered_array,
-                   @array.filter(filter))
+                   @array.filter(filter, @options))
     end
 
     test("Arrow::BooleanArray") do
       filter = Arrow::BooleanArray.new([nil, true, true, false])
       filtered_array = Arrow::BooleanArray.new([nil, false, false])
       assert_equal(filtered_array,
-                   @array.filter(filter))
+                   @array.filter(filter, @options))
     end
 
     test("Arrow::ChunkedArray") do
@@ -100,7 +102,7 @@ class ArrayTest < Test::Unit::TestCase
       filter = Arrow::ChunkedArray.new(chunks)
       filtered_array = Arrow::BooleanArray.new([nil, false, false])
       assert_equal(filtered_array,
-                   @array.filter(filter))
+                   @array.filter(filter, @options))
     end
   end
 

data/test/test-chunked-array.rb

@@ -92,6 +92,8 @@ class ChunkedArrayTest < Test::Unit::TestCase
         Arrow::BooleanArray.new([false, true, false]),
       ]
       @chunked_array = Arrow::ChunkedArray.new(arrays)
+      @options = Arrow::FilterOptions.new
+      @options.null_selection_behavior = :emit_null
     end
 
     test("Array: boolean") do
@@ -102,7 +104,7 @@ class ChunkedArrayTest < Test::Unit::TestCase
       ]
       filtered_chunked_array = Arrow::ChunkedArray.new(chunks)
       assert_equal(filtered_chunked_array,
-                   @chunked_array.filter(filter))
+                   @chunked_array.filter(filter, @options))
     end
 
     test("Arrow::BooleanArray") do
@@ -113,7 +115,7 @@ class ChunkedArrayTest < Test::Unit::TestCase
       ]
       filtered_chunked_array = Arrow::ChunkedArray.new(chunks)
       assert_equal(filtered_chunked_array,
-                   @chunked_array.filter(filter))
+                   @chunked_array.filter(filter, @options))
     end
 
     test("Arrow::ChunkedArray") do
@@ -128,7 +130,7 @@ class ChunkedArrayTest < Test::Unit::TestCase
       ]
       filtered_chunked_array = Arrow::ChunkedArray.new(filtered_chunks)
       assert_equal(filtered_chunked_array,
-                   @chunked_array.filter(filter))
+                   @chunked_array.filter(filter, @options))
     end
   end
 

data/test/test-feather.rb

@@ -18,17 +18,32 @@
 class FeatherTest < Test::Unit::TestCase
   include Helper::Fixture
 
-  def test_save_load
+  def setup
     columns = {
       "message" => Arrow::StringArray.new(["Start", "Crash", "Shutdown"]),
       "is_critical" => Arrow::BooleanArray.new([false, true, false]),
     }
-    table = Arrow::Table.new(columns)
+    @table = Arrow::Table.new(columns)
 
-    output = Tempfile.new(["red-arrow", ".feather"])
-    table.save(output.path)
-    output.close
+    @output = Tempfile.new(["red-arrow", ".feather"])
+    begin
+      yield(@output)
+    rescue
+      @output.close!
+    end
+  end
+
+  def test_default
+    @table.save(@output.path)
+    @output.close
+
+    assert_equal(@table, Arrow::Table.load(@output.path))
+  end
+
+  def test_compression
+    @table.save(@output.path, compression: :zstd)
+    @output.close
 
-    assert_equal(table, Arrow::Table.load(output.path))
+    assert_equal(@table, Arrow::Table.load(@output.path))
   end
 end

data/test/test-schema.rb

@@ -16,6 +16,8 @@
 # under the License.
 
 class SchemaTest < Test::Unit::TestCase
+  include Helper::Omittable
+
   def setup
     @count_field = Arrow::Field.new("count", :uint32)
     @visible_field = Arrow::Field.new("visible", :boolean)
@@ -114,5 +116,19 @@ class SchemaTest < Test::Unit::TestCase
         end
       end
     end
+
+    sub_test_case("#to_s") do
+      test("show_metadata") do
+        require_gi_bindings(3, 4, 2)
+
+        schema = @schema.with_metadata("key" => "value")
+        assert_equal(<<-SCHEMA.chomp, schema.to_s(show_metadata: true))
+count: uint32
+visible: bool
+-- metadata --
+key: value
+        SCHEMA
+      end
+    end
   end
 end

data/test/test-table.rb

@@ -698,9 +698,15 @@ visible: false
   end
 
   sub_test_case("#filter") do
+    def setup
+      super
+      @options = Arrow::FilterOptions.new
+      @options.null_selection_behavior = :emit_null
+    end
+
     test("Array: boolean") do
       filter = [nil, true, true, false, true, false, true, true]
-      assert_equal(<<-TABLE, @table.filter(filter).to_s)
+      assert_equal(<<-TABLE, @table.filter(filter, @options).to_s)
 	count	visible
 0	     	       
 1	    2	false  
@@ -714,7 +720,7 @@ visible: false
     test("Arrow::BooleanArray") do
       array = [nil, true, true, false, true, false, true, true]
       filter = Arrow::BooleanArray.new(array)
-      assert_equal(<<-TABLE, @table.filter(filter).to_s)
+      assert_equal(<<-TABLE, @table.filter(filter, @options).to_s)
 	count	visible
 0	     	       
 1	    2	false  
@@ -732,7 +738,7 @@ visible: false
         Arrow::BooleanArray.new([true, true]),
       ]
       filter = Arrow::ChunkedArray.new(filter_chunks)
-      assert_equal(<<-TABLE, @table.filter(filter).to_s)
+      assert_equal(<<-TABLE, @table.filter(filter, @options).to_s)
 	count	visible
 0	     	       
 1	    2	false  

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-arrow
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.17.0
 platform: ruby
 authors:
 - Apache Arrow Developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-07 00:00:00.000000000 Z
+date: 2020-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extpp
@@ -265,6 +265,7 @@ files:
 - test/fixture/without-header.csv
 - test/helper.rb
 - test/helper/fixture.rb
+- test/helper/omittable.rb
 - test/raw-records/test-basic-arrays.rb
 - test/raw-records/test-dense-union-array.rb
 - test/raw-records/test-list-array.rb
@@ -326,7 +327,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata:
-  msys2_mingw_dependencies: arrow>=0.16.0
+  msys2_mingw_dependencies: arrow>=0.17.0
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -342,77 +343,77 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Red Arrow is the Ruby bindings of Apache Arrow
 test_files:
-- test/test-time64-data-type.rb
-- test/test-feather.rb
-- test/test-decimal128.rb
-- test/test-struct-array.rb
-- test/test-data-type.rb
-- test/test-list-data-type.rb
+- test/test-orc.rb
+- test/test-file-output-stream.rb
 - test/test-dense-union-data-type.rb
-- test/helper.rb
+- test/test-record-batch-builder.rb
 - test/test-record-batch.rb
+- test/test-list-data-type.rb
+- test/test-decimal128.rb
+- test/test-chunked-array.rb
+- test/test-array.rb
+- test/test-null-array.rb
+- test/test-struct-array-builder.rb
 - test/test-table.rb
-- test/run-test.rb
-- test/test-timestamp-array.rb
-- test/test-date32-array.rb
-- test/test-array-builder.rb
-- test/test-date64-array.rb
-- test/test-record-batch-file-reader.rb
 - test/test-decimal128-data-type.rb
-- test/test-time.rb
-- test/test-timestamp-data-type.rb
-- test/test-column.rb
-- test/test-field.rb
-- test/test-decimal128-array.rb
-- test/test-csv-loader.rb
-- test/test-bigdecimal.rb
-- test/test-list-array.rb
-- test/test-time64-array.rb
-- test/test-rolling-window.rb
-- test/test-dictionary-data-type.rb
+- test/fixture/TestOrcFile.test1.orc
 - test/fixture/integer-float.csv
-- test/fixture/without-header-float.csv
 - test/fixture/null-with-double-quote.csv
+- test/fixture/with-header.csv
+- test/fixture/without-header.csv
+- test/fixture/without-header-float.csv
 - test/fixture/with-header-float.csv
-- test/fixture/TestOrcFile.test1.orc
 - test/fixture/null-without-double-quote.csv
-- test/fixture/without-header.csv
-- test/fixture/with-header.csv
 - test/fixture/float-integer.csv
-- test/test-orc.rb
-- test/test-null-array.rb
-- test/test-time32-data-type.rb
-- test/test-struct-data-type.rb
+- test/run-test.rb
 - test/test-group.rb
-- test/test-buffer.rb
-- test/values/test-struct-array.rb
+- test/helper.rb
+- test/test-time64-array.rb
+- test/test-csv-loader.rb
+- test/test-feather.rb
+- test/test-time64-data-type.rb
+- test/values/test-sparse-union-array.rb
 - test/values/test-basic-arrays.rb
-- test/values/test-list-array.rb
 - test/values/test-dense-union-array.rb
-- test/values/test-sparse-union-array.rb
-- test/test-slicer.rb
-- test/test-list-array-builder.rb
-- test/test-sparse-union-data-type.rb
-- test/test-record-batch-builder.rb
-- test/raw-records/test-struct-array.rb
-- test/raw-records/test-table.rb
-- test/raw-records/test-basic-arrays.rb
-- test/raw-records/test-list-array.rb
-- test/raw-records/test-dense-union-array.rb
+- test/values/test-list-array.rb
+- test/values/test-struct-array.rb
+- test/test-struct-data-type.rb
+- test/test-date64-array.rb
 - test/raw-records/test-sparse-union-array.rb
 - test/raw-records/test-multiple-columns.rb
-- test/test-array.rb
-- test/test-file-output-stream.rb
-- test/test-tensor.rb
+- test/raw-records/test-basic-arrays.rb
+- test/raw-records/test-table.rb
+- test/raw-records/test-dense-union-array.rb
+- test/raw-records/test-list-array.rb
+- test/raw-records/test-struct-array.rb
+- test/test-list-array.rb
+- test/test-timestamp-data-type.rb
+- test/test-timestamp-array.rb
+- test/test-data-type.rb
+- test/test-array-builder.rb
+- test/test-time32-data-type.rb
+- test/test-record-batch-file-reader.rb
+- test/test-rolling-window.rb
+- test/test-list-array-builder.rb
+- test/test-slicer.rb
 - test/test-decimal128-array-builder.rb
-- test/test-chunked-array.rb
 - test/test-schema.rb
-- test/test-struct-array-builder.rb
-- test/helper/fixture.rb
+- test/test-time.rb
+- test/test-column.rb
+- test/test-bigdecimal.rb
+- test/test-sparse-union-data-type.rb
+- test/test-tensor.rb
 - test/test-time32-array.rb
+- test/test-buffer.rb
+- test/test-decimal128-array.rb
+- test/helper/omittable.rb
+- test/helper/fixture.rb
+- test/test-dictionary-data-type.rb
+- test/test-date32-array.rb
+- test/test-field.rb
+- test/test-struct-array.rb