red-parquet 0.17.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11d3c62ada89745e9665423932e1161866dfe8cc01adbee2623b22bdbcd52a72
-  data.tar.gz: a7818ef72443c282b14fedb5d2fd39118ed06227ffd270fa8758fb1ae232e28d
+  metadata.gz: fe7b0bb0ec8f4a2420a2ab1001f2b296e7ceb7946e06aa3abe523163f67c7349
+  data.tar.gz: 3714403cabdb00f02dcd815a7db91141882de6192bd3b4b1cd12c213ed31c08c
 SHA512:
-  metadata.gz: c401ddccc2252fb4521ba99969bea4f6534797ac5f682084837c72b3ed7ee5a46d386512153f9f0c12ed60de27e7c32530eb448e7ed046fdbe3e92f7538819e1
-  data.tar.gz: e682d30bf304aead6172dd984dd786b62b3e85916ff6f75ef419f1ccf1a870d5540f1be95f46b6f5ca196ffe0eda44fe754d24e7888ee3b30b2d56e2708150a0
+  metadata.gz: 6a071b5bed3343f0500f42fc242327d08634f08d3443a167c6b46d65b0b2645b63c24b21c4272d082bf0548bb4249250c27854370d672b66d6822d89f37db323
+  data.tar.gz: 4745b76798fbeab130a36235a14ba8d694597c789a3b284cc8cb71108d86c4e1c1ae2a9c45f994aa44624bc8b16bdd62606aa837fe95214d4bd4d8d6f06de2e4

data/lib/parquet/arrow-table-savable.rb

@@ -19,9 +19,25 @@ module Parquet
   module ArrowTableSavable
     private
     def save_as_parquet
+      properties = WriterProperties.new
+      @options.each do |key, value|
+        next if value.nil?
+        set_method_name = "set_#{key}"
+        next unless properties.respond_to?(set_method_name)
+        case value
+        when ::Array, ::Hash
+          value.each do |path, v|
+            properties.__send__(set_method_name, v, path)
+          end
+        else
+          properties.__send__(set_method_name, value)
+        end
+      end
       chunk_size = @options[:chunk_size] || 1024 # TODO
-      open_output_stream do |output|
-        Parquet::ArrowFileWriter.open(@table.schema, output) do |writer|
+      open_raw_output_stream do |output|
+        ArrowFileWriter.open(@table.schema,
+                             output,
+                             properties) do |writer|
           writer.write_table(@table, chunk_size)
         end
       end

data/lib/parquet/loader.rb

@@ -31,6 +31,7 @@ module Parquet
     def require_libraries
       require "parquet/arrow-table-loadable"
       require "parquet/arrow-table-savable"
+      require "parquet/writer-properties"
     end
 
     def load_object_info(info)

data/lib/parquet/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Parquet
-  VERSION = "0.17.1"
+  VERSION = "1.0.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/lib/parquet/writer-properties.rb

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Parquet
+  class WriterProperties
+    def set_dictionary(enable, path=nil)
+      if enable
+        enable_dictionary(path)
+      else
+        disable_dictionary(path)
+      end
+    end
+  end
+end

data/test/test-arrow-table.rb

@@ -19,7 +19,8 @@ class TestArrowTableReader < Test::Unit::TestCase
   def setup
     @count_field = Arrow::Field.new("count", :uint8)
     @visible_field = Arrow::Field.new("visible", :boolean)
-    schema = Arrow::Schema.new([@count_field, @visible_field])
+    @label_field = Arrow::Field.new("label", :string)
+    schema = Arrow::Schema.new([@count_field, @visible_field, @label_field])
     count_arrays = [
       Arrow::UInt8Array.new([1, 2]),
       Arrow::UInt8Array.new([4, 8, 16]),
@@ -33,16 +34,30 @@ class TestArrowTableReader < Test::Unit::TestCase
       Arrow::BooleanArray.new([nil]),
       Arrow::BooleanArray.new([nil]),
     ]
+    label_arrays = [
+      Arrow::StringArray.new(["a"]),
+      Arrow::StringArray.new(["b", "c"]),
+      Arrow::StringArray.new(["d", nil, nil]),
+      Arrow::StringArray.new(["e", "f"]),
+    ]
     @count_array = Arrow::ChunkedArray.new(count_arrays)
     @visible_array = Arrow::ChunkedArray.new(visible_arrays)
-    @table = Arrow::Table.new(schema, [@count_array, @visible_array])
+    @label_array = Arrow::ChunkedArray.new(label_arrays)
+    @table = Arrow::Table.new(schema,
+                              [@count_array, @visible_array, @label_array])
+
+    @output = Tempfile.open(["red-parquet", ".parquet"])
+    begin
+      yield(@output)
+    ensure
+      @output.close!
+    end
   end
 
   def test_save_load_path
-    tempfile = Tempfile.open(["red-parquet", ".parquet"])
-    @table.save(tempfile.path)
+    @table.save(@output.path)
     assert do
-      @table.equal_metadata(Arrow::Table.load(tempfile.path), false)
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
     end
   end
 
@@ -53,4 +68,32 @@ class TestArrowTableReader < Test::Unit::TestCase
       @table.equal_metadata(Arrow::Table.load(buffer, format: :parquet), false)
     end
   end
+
+  def test_save_load_compression
+    @table.save(@output.path, compression: :zstd)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_compression_path
+    @table.save(@output.path, compression: {"count" => :zstd})
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary
+    @table.save(@output.path, dictionary: false)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary_path
+    @table.save(@output.path, dictionary: [["label", false]])
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-parquet
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Apache Arrow Developers
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-18 00:00:00.000000000 Z
+date: 2020-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.17.1
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.17.1
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -85,6 +85,7 @@ files:
 - lib/parquet/arrow-table-savable.rb
 - lib/parquet/loader.rb
 - lib/parquet/version.rb
+- lib/parquet/writer-properties.rb
 - red-parquet.gemspec
 - test/helper.rb
 - test/run-test.rb
@@ -93,7 +94,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Red Parquet is the Ruby bindings of Apache Parquet
 test_files: