red-parquet 0.17.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f70b51b41d1f3270363f9492b9f797d9eb2c3bb43ec507766874390b0e8970a8
-  data.tar.gz: bb36f2b4640b2a6e1746e08f362b3fc84af6fd76d4806ed99ee7cbf67a7f5dc9
+  metadata.gz: 6bc3ea63383a2a988aac1fd3596007c44427e226447df1610985deb4752b2142
+  data.tar.gz: 2f36454279a9f304df19279e429d87f2e2fe76314fb8523c1917ba443d23dc61
 SHA512:
-  metadata.gz: 99b3b1008d9f1c43732f25d04e3b3e2633e4cd4b9f0034573d1b5e399bf6680eff3b645b40c60bce92c6bd9ae7ac60bfd4b394d17f2c0011804da32bef10350b
-  data.tar.gz: 51f70aa5c9c36f336c7c4f14d02b61e12e6a31937a0c22aa5251083b22f6f8eee14246c983a5a8ca9031c22d4ef0e4156ef0a970505e1f7c9b298efa6d11b50f
+  metadata.gz: 8d2f7f3fd3e9034fbcac06f37377b14093bda97e073b3ce7d302e78e88814c5496a3eb68f7a6e942ded966266badc89608ac38867cedb6e000bd4c8167184d38
+  data.tar.gz: 72ca6746ff2aa8435e5511e6b423ba365b1b2f007f21d5760895117203cab9f0d68a3e95d56f051f207c90b6875c7637813c67a76f9610f3ef00187f403b642f

data/dependency-check/Rakefile

@@ -19,6 +19,7 @@
 
 require "pkg-config"
 require "native-package-installer"
+require_relative "../lib/parquet/version"
 
 case RUBY_PLATFORM
 when /mingw|mswin/
@@ -33,7 +34,10 @@ end
 namespace :dependency do
   desc "Check dependency"
   task :check do
-    unless PKGConfig.check_version?("parquet-glib")
+    unless PKGConfig.check_version?("parquet-glib",
+                                    Parquet::Version::MAJOR,
+                                    Parquet::Version::MINOR,
+                                    Parquet::Version::MICRO)
       unless NativePackageInstaller.install(:debian => "libparquet-glib-dev",
                                             :redhat => "parquet-glib-devel")
         exit(false)

data/lib/parquet/arrow-table-savable.rb

@@ -19,9 +19,25 @@ module Parquet
   module ArrowTableSavable
     private
     def save_as_parquet
-      chunk_size = @options[:chunk_size] || 1024 # TODO
-      open_output_stream do |output|
-        Parquet::ArrowFileWriter.open(@table.schema, output) do |writer|
+      properties = WriterProperties.new
+      @options.each do |key, value|
+        next if value.nil?
+        set_method_name = "set_#{key}"
+        next unless properties.respond_to?(set_method_name)
+        case value
+        when ::Array, ::Hash
+          value.each do |path, v|
+            properties.__send__(set_method_name, v, path)
+          end
+        else
+          properties.__send__(set_method_name, value)
+        end
+      end
+      chunk_size = @options[:chunk_size] || @table.n_rows
+      open_raw_output_stream do |output|
+        ArrowFileWriter.open(@table.schema,
+                             output,
+                             properties) do |writer|
           writer.write_table(@table, chunk_size)
         end
       end

data/lib/parquet/loader.rb

@@ -31,6 +31,7 @@ module Parquet
     def require_libraries
       require "parquet/arrow-table-loadable"
       require "parquet/arrow-table-savable"
+      require "parquet/writer-properties"
     end
 
     def load_object_info(info)

data/lib/parquet/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Parquet
-  VERSION = "0.17.0"
+  VERSION = "3.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/lib/parquet/writer-properties.rb

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Parquet
+  class WriterProperties
+    def set_dictionary(enable, path=nil)
+      if enable
+        enable_dictionary(path)
+      else
+        disable_dictionary(path)
+      end
+    end
+  end
+end

data/test/test-arrow-table.rb

@@ -19,7 +19,8 @@ class TestArrowTableReader < Test::Unit::TestCase
   def setup
     @count_field = Arrow::Field.new("count", :uint8)
     @visible_field = Arrow::Field.new("visible", :boolean)
-    schema = Arrow::Schema.new([@count_field, @visible_field])
+    @label_field = Arrow::Field.new("label", :string)
+    schema = Arrow::Schema.new([@count_field, @visible_field, @label_field])
     count_arrays = [
       Arrow::UInt8Array.new([1, 2]),
       Arrow::UInt8Array.new([4, 8, 16]),
@@ -33,16 +34,30 @@ class TestArrowTableReader < Test::Unit::TestCase
       Arrow::BooleanArray.new([nil]),
       Arrow::BooleanArray.new([nil]),
     ]
+    label_arrays = [
+      Arrow::StringArray.new(["a"]),
+      Arrow::StringArray.new(["b", "c"]),
+      Arrow::StringArray.new(["d", nil, nil]),
+      Arrow::StringArray.new(["e", "f"]),
+    ]
     @count_array = Arrow::ChunkedArray.new(count_arrays)
     @visible_array = Arrow::ChunkedArray.new(visible_arrays)
-    @table = Arrow::Table.new(schema, [@count_array, @visible_array])
+    @label_array = Arrow::ChunkedArray.new(label_arrays)
+    @table = Arrow::Table.new(schema,
+                              [@count_array, @visible_array, @label_array])
+
+    @output = Tempfile.open(["red-parquet", ".parquet"])
+    begin
+      yield(@output)
+    ensure
+      @output.close!
+    end
   end
 
   def test_save_load_path
-    tempfile = Tempfile.open(["red-parquet", ".parquet"])
-    @table.save(tempfile.path)
+    @table.save(@output.path)
     assert do
-      @table.equal_metadata(Arrow::Table.load(tempfile.path), false)
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
     end
   end
 
@@ -53,4 +68,32 @@ class TestArrowTableReader < Test::Unit::TestCase
       @table.equal_metadata(Arrow::Table.load(buffer, format: :parquet), false)
     end
   end
+
+  def test_save_load_compression
+    @table.save(@output.path, compression: :zstd)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_compression_path
+    @table.save(@output.path, compression: {"count" => :zstd})
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary
+    @table.save(@output.path, dictionary: false)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary_path
+    @table.save(@output.path, dictionary: [["label", false]])
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-parquet
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-20 00:00:00.000000000 Z
+date: 2021-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.17.0
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.17.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -85,6 +85,7 @@ files:
 - lib/parquet/arrow-table-savable.rb
 - lib/parquet/loader.rb
 - lib/parquet/version.rb
+- lib/parquet/writer-properties.rb
 - red-parquet.gemspec
 - test/helper.rb
 - test/run-test.rb
@@ -108,11 +109,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.5
 signing_key: 
 specification_version: 4
 summary: Red Parquet is the Ruby bindings of Apache Parquet
 test_files:
-- test/run-test.rb
 - test/helper.rb
+- test/run-test.rb
 - test/test-arrow-table.rb