red-parquet 0.17.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11d3c62ada89745e9665423932e1161866dfe8cc01adbee2623b22bdbcd52a72
-  data.tar.gz: a7818ef72443c282b14fedb5d2fd39118ed06227ffd270fa8758fb1ae232e28d
+  metadata.gz: d6b91c2b8aab70448c3aba923900385ebd9218d69760f8baf5a01dd1ad9977ef
+  data.tar.gz: 90a3980cada0d03897167cb53b1f9197bea059b72431c60ace09f33105d5f61e
 SHA512:
-  metadata.gz: c401ddccc2252fb4521ba99969bea4f6534797ac5f682084837c72b3ed7ee5a46d386512153f9f0c12ed60de27e7c32530eb448e7ed046fdbe3e92f7538819e1
-  data.tar.gz: e682d30bf304aead6172dd984dd786b62b3e85916ff6f75ef419f1ccf1a870d5540f1be95f46b6f5ca196ffe0eda44fe754d24e7888ee3b30b2d56e2708150a0
+  metadata.gz: 746f209269f4e2d273636d72de17b0c5a284f9a0adb244af8f94a558e0e5d93f77f743a2b8df8d05d313a02f4aa3110cf0c83b27e5176d5213cef7abae7cccd8
+  data.tar.gz: f46ffe92f4c692f46e4577345453e89a0063e897f7a3c669f1e6d83d785136cbab02eb8aaee069a18fd5cb6c8bddf8251e053b2765af9bd72dc598f235fb6afc

data/dependency-check/Rakefile

@@ -19,6 +19,7 @@
 
 require "pkg-config"
 require "native-package-installer"
+require_relative "../lib/parquet/version"
 
 case RUBY_PLATFORM
 when /mingw|mswin/
@@ -33,7 +34,10 @@ end
 namespace :dependency do
   desc "Check dependency"
   task :check do
-    unless PKGConfig.check_version?("parquet-glib")
+    unless PKGConfig.check_version?("parquet-glib",
+                                    Parquet::Version::MAJOR,
+                                    Parquet::Version::MINOR,
+                                    Parquet::Version::MICRO)
       unless NativePackageInstaller.install(:debian => "libparquet-glib-dev",
                                             :redhat => "parquet-glib-devel")
         exit(false)

data/lib/parquet/arrow-table-savable.rb

@@ -19,9 +19,25 @@ module Parquet
   module ArrowTableSavable
     private
     def save_as_parquet
-      chunk_size = @options[:chunk_size] || 1024 # TODO
-      open_output_stream do |output|
-        Parquet::ArrowFileWriter.open(@table.schema, output) do |writer|
+      properties = WriterProperties.new
+      @options.each do |key, value|
+        next if value.nil?
+        set_method_name = "set_#{key}"
+        next unless properties.respond_to?(set_method_name)
+        case value
+        when ::Array, ::Hash
+          value.each do |path, v|
+            properties.__send__(set_method_name, v, path)
+          end
+        else
+          properties.__send__(set_method_name, value)
+        end
+      end
+      chunk_size = @options[:chunk_size] || @table.n_rows
+      open_raw_output_stream do |output|
+        ArrowFileWriter.open(@table.schema,
+                             output,
+                             properties) do |writer|
           writer.write_table(@table, chunk_size)
         end
       end

data/lib/parquet/loader.rb

@@ -31,6 +31,7 @@ module Parquet
     def require_libraries
       require "parquet/arrow-table-loadable"
       require "parquet/arrow-table-savable"
+      require "parquet/writer-properties"
     end
 
     def load_object_info(info)

data/lib/parquet/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Parquet
-  VERSION = "0.17.1"
+  VERSION = "4.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/lib/parquet/writer-properties.rb

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Parquet
+  class WriterProperties
+    def set_dictionary(enable, path=nil)
+      if enable
+        enable_dictionary(path)
+      else
+        disable_dictionary(path)
+      end
+    end
+  end
+end

data/test/test-arrow-table.rb

@@ -19,7 +19,8 @@ class TestArrowTableReader < Test::Unit::TestCase
   def setup
     @count_field = Arrow::Field.new("count", :uint8)
     @visible_field = Arrow::Field.new("visible", :boolean)
-    schema = Arrow::Schema.new([@count_field, @visible_field])
+    @label_field = Arrow::Field.new("label", :string)
+    schema = Arrow::Schema.new([@count_field, @visible_field, @label_field])
     count_arrays = [
       Arrow::UInt8Array.new([1, 2]),
       Arrow::UInt8Array.new([4, 8, 16]),
@@ -33,16 +34,30 @@ class TestArrowTableReader < Test::Unit::TestCase
       Arrow::BooleanArray.new([nil]),
       Arrow::BooleanArray.new([nil]),
     ]
+    label_arrays = [
+      Arrow::StringArray.new(["a"]),
+      Arrow::StringArray.new(["b", "c"]),
+      Arrow::StringArray.new(["d", nil, nil]),
+      Arrow::StringArray.new(["e", "f"]),
+    ]
     @count_array = Arrow::ChunkedArray.new(count_arrays)
     @visible_array = Arrow::ChunkedArray.new(visible_arrays)
-    @table = Arrow::Table.new(schema, [@count_array, @visible_array])
+    @label_array = Arrow::ChunkedArray.new(label_arrays)
+    @table = Arrow::Table.new(schema,
+                              [@count_array, @visible_array, @label_array])
+
+    @output = Tempfile.open(["red-parquet", ".parquet"])
+    begin
+      yield(@output)
+    ensure
+      @output.close!
+    end
   end
 
   def test_save_load_path
-    tempfile = Tempfile.open(["red-parquet", ".parquet"])
-    @table.save(tempfile.path)
+    @table.save(@output.path)
     assert do
-      @table.equal_metadata(Arrow::Table.load(tempfile.path), false)
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
     end
   end
 
@@ -53,4 +68,32 @@ class TestArrowTableReader < Test::Unit::TestCase
       @table.equal_metadata(Arrow::Table.load(buffer, format: :parquet), false)
     end
   end
+
+  def test_save_load_compression
+    @table.save(@output.path, compression: :zstd)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_compression_path
+    @table.save(@output.path, compression: {"count" => :zstd})
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary
+    @table.save(@output.path, dictionary: false)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary_path
+    @table.save(@output.path, dictionary: [["label", false]])
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-parquet
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-18 00:00:00.000000000 Z
+date: 2021-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.17.1
+        version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.17.1
+        version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -85,6 +85,7 @@ files:
 - lib/parquet/arrow-table-savable.rb
 - lib/parquet/loader.rb
 - lib/parquet/version.rb
+- lib/parquet/writer-properties.rb
 - red-parquet.gemspec
 - test/helper.rb
 - test/run-test.rb
@@ -93,7 +94,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -108,11 +109,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: Red Parquet is the Ruby bindings of Apache Parquet
 test_files:
-- test/run-test.rb
 - test/helper.rb
+- test/run-test.rb
 - test/test-arrow-table.rb