red-parquet 0.16.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8d253cc5855c8c6a474a8258ffbb07ba1a8d1a232cdb15930d0022b9ff0fc94
-  data.tar.gz: d44a86bf31748f8f19e68a3eabab7945ab331769b30c72dd5d804bc5b638b0c4
+  metadata.gz: be8752047a8d28fdc406e270e00bed959dee407eaca6a1d9926e9d62ee80ae5b
+  data.tar.gz: b09324d81dcbf726b9115f411e881b8706339d4e7b0755ca47fb02ecc5af4758
 SHA512:
-  metadata.gz: 907010ccfd3a11245e668b447db3df753e31f91e31576d7cf125cb31907ae14fad5bfea75c2b3220092a971b88dba08383a13ae5040267100abfc53865993cf8
-  data.tar.gz: 75891ebbbc5605851e167333550896abfeef809c76b5dc440e32fbe755f94c2a985fb81226511a1cb400ca3b6822c600f12ea419f08f2dde2d77f319ef0b70ea
+  metadata.gz: 5249d476253ff922f4552032ab87fe251b428698314a1a45160601b9f5ccac92af6337934509c5cfd8a9ee1c463fb1b8824215692c216aa5c65cb0a40d4d4cde
+  data.tar.gz: 7c5009575fb4c0039813fdf6d0b3ce53eeb32c2760415cc397391bd74bf89589cf45421c060261b097eca7173ee69cb4dc73be97fbcf9b8a712addf2e37155f8

data/Rakefile

@@ -30,10 +30,12 @@ release_task.prerequisites.replace(["build", "release:rubygem_push"])
 
 desc "Run tests"
 task :test do
-  cd("dependency-check") do
-    ruby("-S", "rake")
+  cd(base_dir) do
+    cd("dependency-check") do
+      ruby("-S", "rake")
+    end
+    ruby("test/run-test.rb")
   end
-  ruby("test/run-test.rb")
 end
 
 task default: :test

data/dependency-check/Rakefile

@@ -19,6 +19,7 @@
 
 require "pkg-config"
 require "native-package-installer"
+require_relative "../lib/parquet/version"
 
 case RUBY_PLATFORM
 when /mingw|mswin/
@@ -33,7 +34,10 @@ end
 namespace :dependency do
   desc "Check dependency"
   task :check do
-    unless PKGConfig.check_version?("parquet-glib")
+    unless PKGConfig.check_version?("parquet-glib",
+                                    Parquet::Version::MAJOR,
+                                    Parquet::Version::MINOR,
+                                    Parquet::Version::MICRO)
       unless NativePackageInstaller.install(:debian => "libparquet-glib-dev",
                                             :redhat => "parquet-glib-devel")
         exit(false)

data/lib/parquet/arrow-table-savable.rb

@@ -19,9 +19,25 @@ module Parquet
   module ArrowTableSavable
     private
     def save_as_parquet
-      chunk_size = @options[:chunk_size] || 1024 # TODO
-      open_output_stream do |output|
-        Parquet::ArrowFileWriter.open(@table.schema, output) do |writer|
+      properties = WriterProperties.new
+      @options.each do |key, value|
+        next if value.nil?
+        set_method_name = "set_#{key}"
+        next unless properties.respond_to?(set_method_name)
+        case value
+        when ::Array, ::Hash
+          value.each do |path, v|
+            properties.__send__(set_method_name, v, path)
+          end
+        else
+          properties.__send__(set_method_name, value)
+        end
+      end
+      chunk_size = @options[:chunk_size] || @table.n_rows
+      open_raw_output_stream do |output|
+        ArrowFileWriter.open(@table.schema,
+                             output,
+                             properties) do |writer|
           writer.write_table(@table, chunk_size)
         end
       end

data/lib/parquet/loader.rb

@@ -31,6 +31,7 @@ module Parquet
     def require_libraries
       require "parquet/arrow-table-loadable"
       require "parquet/arrow-table-savable"
+      require "parquet/writer-properties"
     end
 
     def load_object_info(info)

data/lib/parquet/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Parquet
-  VERSION = "0.16.0"
+  VERSION = "2.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/lib/parquet/writer-properties.rb

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Parquet
+  class WriterProperties
+    def set_dictionary(enable, path=nil)
+      if enable
+        enable_dictionary(path)
+      else
+        disable_dictionary(path)
+      end
+    end
+  end
+end

data/test/run-test.rb

@@ -34,7 +34,14 @@ test_dir = base_dir + "test"
 arrow_lib_dir = arrow_base_dir + "lib"
 arrow_ext_dir = arrow_base_dir + "ext" + "arrow"
 
-$LOAD_PATH.unshift(arrow_ext_dir.to_s)
+build_dir = ENV["BUILD_DIR"]
+if build_dir
+  arrow_build_dir = Pathname.new(build_dir) + "red-arrow"
+else
+  arrow_build_dir = arrow_ext_dir
+end
+
+$LOAD_PATH.unshift(arrow_build_dir.to_s)
 $LOAD_PATH.unshift(arrow_lib_dir.to_s)
 $LOAD_PATH.unshift(lib_dir.to_s)
 

data/test/test-arrow-table.rb

@@ -19,7 +19,8 @@ class TestArrowTableReader < Test::Unit::TestCase
   def setup
     @count_field = Arrow::Field.new("count", :uint8)
     @visible_field = Arrow::Field.new("visible", :boolean)
-    schema = Arrow::Schema.new([@count_field, @visible_field])
+    @label_field = Arrow::Field.new("label", :string)
+    schema = Arrow::Schema.new([@count_field, @visible_field, @label_field])
     count_arrays = [
       Arrow::UInt8Array.new([1, 2]),
       Arrow::UInt8Array.new([4, 8, 16]),
@@ -33,20 +34,66 @@ class TestArrowTableReader < Test::Unit::TestCase
       Arrow::BooleanArray.new([nil]),
       Arrow::BooleanArray.new([nil]),
     ]
+    label_arrays = [
+      Arrow::StringArray.new(["a"]),
+      Arrow::StringArray.new(["b", "c"]),
+      Arrow::StringArray.new(["d", nil, nil]),
+      Arrow::StringArray.new(["e", "f"]),
+    ]
     @count_array = Arrow::ChunkedArray.new(count_arrays)
     @visible_array = Arrow::ChunkedArray.new(visible_arrays)
-    @table = Arrow::Table.new(schema, [@count_array, @visible_array])
+    @label_array = Arrow::ChunkedArray.new(label_arrays)
+    @table = Arrow::Table.new(schema,
+                              [@count_array, @visible_array, @label_array])
+
+    @output = Tempfile.open(["red-parquet", ".parquet"])
+    begin
+      yield(@output)
+    ensure
+      @output.close!
+    end
   end
 
   def test_save_load_path
-    tempfile = Tempfile.open(["red-parquet", ".parquet"])
-    @table.save(tempfile.path)
-    assert_equal(@table, Arrow::Table.load(tempfile.path))
+    @table.save(@output.path)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
   end
 
   def test_save_load_buffer
     buffer = Arrow::ResizableBuffer.new(1024)
     @table.save(buffer, format: :parquet)
-    assert_equal(@table, Arrow::Table.load(buffer, format: :parquet))
+    assert do
+      @table.equal_metadata(Arrow::Table.load(buffer, format: :parquet), false)
+    end
+  end
+
+  def test_save_load_compression
+    @table.save(@output.path, compression: :zstd)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_compression_path
+    @table.save(@output.path, compression: {"count" => :zstd})
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary
+    @table.save(@output.path, dictionary: false)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary_path
+    @table.save(@output.path, dictionary: [["label", false]])
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-parquet
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-07 00:00:00.000000000 Z
+date: 2020-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.16.0
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.16.0
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -85,6 +85,7 @@ files:
 - lib/parquet/arrow-table-savable.rb
 - lib/parquet/loader.rb
 - lib/parquet/version.rb
+- lib/parquet/writer-properties.rb
 - red-parquet.gemspec
 - test/helper.rb
 - test/run-test.rb
@@ -93,7 +94,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -108,12 +109,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Red Parquet is the Ruby bindings of Apache Parquet
 test_files:
-- test/helper.rb
 - test/run-test.rb
+- test/helper.rb
 - test/test-arrow-table.rb