red-parquet 0.15.1 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 699be2bb00f42f7dc5d021fa0526fff161c53bbe54439a2d9e437bedb3b4bde9
-  data.tar.gz: 1d8bd764218b5a761ad95d1847a73d987d76ae8c1dc887a9aa9011d54b96d89b
+  metadata.gz: 9a50764732fd4ddaf7563956b624d09ef09a72ea4fa64c9388ce64033831f223
+  data.tar.gz: edee5db1aa5a76071d1128384dcbeb1af6755fcaf3004cd2d24474f25b9193ad
 SHA512:
-  metadata.gz: 42492ad141078327bd03a484dd164a7a07e3d3c1ab54e842ac046e6d9fd0e206a4f0c926730076f4ff55c5c5227db29181b0861f45a6d25c48e58b077b6df75a
-  data.tar.gz: 87a62a96ed642ef474495553459e6d405fd1d8e483709c19a43df0c5d845ce6ce236405b36f27dd30575616199e3e52a70a270d2af99e7b402ce6c01257f7f2c
+  metadata.gz: 6f9f51ff3af67261bbf97c36d954995e97daa6f4df7413f85106878d13091515f49831d81b4c2fe366495e05e0eedf5dd0d62fec46cab206c7420c128ac56576
+  data.tar.gz: 513ece4c28ec862f7694349a0f4740c1a86ba3cb39fbb5d262aa188cf471dbeb40fdb30aa1ac1edf80a1efa6bd058ccd99e0b9ad3b33f99ebda6af402b68bc40

data/README.md

@@ -27,7 +27,7 @@ Red Parquet is the Ruby bindings of Apache Parquet. Red Parquet is based on GObj
 
 Red Parquet uses [Apache Parquet GLib](https://github.com/apache/arrow/tree/master/c_glib/parquet-glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Parquet.
 
-Apache Parquet GLib is a C wrapper for [Apache Parquet C++](https://github.com/apache/arrow/tree/master/cpp/parquet). GObject Introspection can't use Apache Parquet C++ directly. Apache Parquet GLib is a bridge between Apache Parquet C++ and GObject Introspection.
+Apache Parquet GLib is a C wrapper for [Apache Parquet C++](https://github.com/apache/arrow/tree/master/cpp/src/parquet). GObject Introspection can't use Apache Parquet C++ directly. Apache Parquet GLib is a bridge between Apache Parquet C++ and GObject Introspection.
 
 gobject-introspection gem is a Ruby bindings of GObject Introspection. Red Parquet uses GObject Introspection via gobject-introspection gem.
 

data/Rakefile

@@ -30,10 +30,12 @@ release_task.prerequisites.replace(["build", "release:rubygem_push"])
 
 desc "Run tests"
 task :test do
-  cd("dependency-check") do
-    ruby("-S", "rake")
+  cd(base_dir) do
+    cd("dependency-check") do
+      ruby("-S", "rake")
+    end
+    ruby("test/run-test.rb")
   end
-  ruby("test/run-test.rb")
 end
 
 task default: :test

data/lib/parquet/arrow-table-savable.rb

@@ -19,9 +19,25 @@ module Parquet
   module ArrowTableSavable
     private
     def save_as_parquet
+      properties = WriterProperties.new
+      @options.each do |key, value|
+        next if value.nil?
+        set_method_name = "set_#{key}"
+        next unless properties.respond_to?(set_method_name)
+        case value
+        when ::Array, ::Hash
+          value.each do |path, v|
+            properties.__send__(set_method_name, v, path)
+          end
+        else
+          properties.__send__(set_method_name, value)
+        end
+      end
       chunk_size = @options[:chunk_size] || 1024 # TODO
-      open_output_stream do |output|
-        Parquet::ArrowFileWriter.open(@table.schema, output) do |writer|
+      open_raw_output_stream do |output|
+        ArrowFileWriter.open(@table.schema,
+                             output,
+                             properties) do |writer|
           writer.write_table(@table, chunk_size)
         end
       end

data/lib/parquet/loader.rb

@@ -31,6 +31,7 @@ module Parquet
     def require_libraries
       require "parquet/arrow-table-loadable"
       require "parquet/arrow-table-savable"
+      require "parquet/writer-properties"
     end
 
     def load_object_info(info)

data/lib/parquet/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Parquet
-  VERSION = "0.15.1"
+  VERSION = "1.0.1"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/lib/parquet/writer-properties.rb

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Parquet
+  class WriterProperties
+    def set_dictionary(enable, path=nil)
+      if enable
+        enable_dictionary(path)
+      else
+        disable_dictionary(path)
+      end
+    end
+  end
+end

data/test/run-test.rb

@@ -34,7 +34,14 @@ test_dir = base_dir + "test"
 arrow_lib_dir = arrow_base_dir + "lib"
 arrow_ext_dir = arrow_base_dir + "ext" + "arrow"
 
-$LOAD_PATH.unshift(arrow_ext_dir.to_s)
+build_dir = ENV["BUILD_DIR"]
+if build_dir
+  arrow_build_dir = Pathname.new(build_dir) + "red-arrow"
+else
+  arrow_build_dir = arrow_ext_dir
+end
+
+$LOAD_PATH.unshift(arrow_build_dir.to_s)
 $LOAD_PATH.unshift(arrow_lib_dir.to_s)
 $LOAD_PATH.unshift(lib_dir.to_s)
 

data/test/test-arrow-table.rb

@@ -19,7 +19,8 @@ class TestArrowTableReader < Test::Unit::TestCase
   def setup
     @count_field = Arrow::Field.new("count", :uint8)
     @visible_field = Arrow::Field.new("visible", :boolean)
-    schema = Arrow::Schema.new([@count_field, @visible_field])
+    @label_field = Arrow::Field.new("label", :string)
+    schema = Arrow::Schema.new([@count_field, @visible_field, @label_field])
     count_arrays = [
       Arrow::UInt8Array.new([1, 2]),
       Arrow::UInt8Array.new([4, 8, 16]),
@@ -33,20 +34,66 @@ class TestArrowTableReader < Test::Unit::TestCase
       Arrow::BooleanArray.new([nil]),
       Arrow::BooleanArray.new([nil]),
     ]
+    label_arrays = [
+      Arrow::StringArray.new(["a"]),
+      Arrow::StringArray.new(["b", "c"]),
+      Arrow::StringArray.new(["d", nil, nil]),
+      Arrow::StringArray.new(["e", "f"]),
+    ]
     @count_array = Arrow::ChunkedArray.new(count_arrays)
     @visible_array = Arrow::ChunkedArray.new(visible_arrays)
-    @table = Arrow::Table.new(schema, [@count_array, @visible_array])
+    @label_array = Arrow::ChunkedArray.new(label_arrays)
+    @table = Arrow::Table.new(schema,
+                              [@count_array, @visible_array, @label_array])
+
+    @output = Tempfile.open(["red-parquet", ".parquet"])
+    begin
+      yield(@output)
+    ensure
+      @output.close!
+    end
   end
 
   def test_save_load_path
-    tempfile = Tempfile.open(["red-parquet", ".parquet"])
-    @table.save(tempfile.path)
-    assert_equal(@table, Arrow::Table.load(tempfile.path))
+    @table.save(@output.path)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
   end
 
   def test_save_load_buffer
     buffer = Arrow::ResizableBuffer.new(1024)
     @table.save(buffer, format: :parquet)
-    assert_equal(@table, Arrow::Table.load(buffer, format: :parquet))
+    assert do
+      @table.equal_metadata(Arrow::Table.load(buffer, format: :parquet), false)
+    end
+  end
+
+  def test_save_load_compression
+    @table.save(@output.path, compression: :zstd)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_compression_path
+    @table.save(@output.path, compression: {"count" => :zstd})
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary
+    @table.save(@output.path, dictionary: false)
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
+  end
+
+  def test_save_load_dictionary_path
+    @table.save(@output.path, dictionary: [["label", false]])
+    assert do
+      @table.equal_metadata(Arrow::Table.load(@output.path), false)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-parquet
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 1.0.1
 platform: ruby
 authors:
 - Apache Arrow Developers
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-01 00:00:00.000000000 Z
+date: 2020-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.15.1
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.15.1
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -85,6 +85,7 @@ files:
 - lib/parquet/arrow-table-savable.rb
 - lib/parquet/loader.rb
 - lib/parquet/version.rb
+- lib/parquet/writer-properties.rb
 - red-parquet.gemspec
 - test/helper.rb
 - test/run-test.rb
@@ -93,7 +94,7 @@ homepage: https://arrow.apache.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -108,8 +109,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Red Parquet is the Ruby bindings of Apache Parquet
 test_files: