red-parquet 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9014915c817a55e329cb0f1c7a977d5dc5a7e03d
-  data.tar.gz: bbbf9ac9ffb74dabc7d04e3092c48669ca9185ad
+  metadata.gz: f182f603bf41836b43670bbb064b30570ed88270
+  data.tar.gz: 3707be9a6e4415441e91a5bb200225487c41123f
 SHA512:
-  metadata.gz: 504364ef1ec483415ef4fa7d133421acef233d25c4fa6a41984c34b4d33cec0d3c3d2de60234631533a271591e3048f26fdb6d7ed9da4444f4129ccf0efc785b
-  data.tar.gz: 921218362e1a7942c4556325316557ad6c879408d7a6320a987881c7e162f29921c31716f7dc77f113b83a138040dc38117c1cce23734048304f23841bbe48ec
+  metadata.gz: 977fa271681371c5e0a827ac55287d7dd3431649f417b61993852fcac0fadda05c97dc9283933be993b98f3c36c3486f457e292b7f35fcccc25ce1d8f3b1ab89
+  data.tar.gz: 9e44068de2e857804b6ea3d3d25e0c1f2c17c920bc26271092818f53bb61e88c0f70775a866b8f0d31503a17f40e1e549bbbe322b94a45eb408ba4875b1b886d

data/doc/text/news.md

@@ -1,5 +1,13 @@
 # News
 
+## 0.0.2 - 2018-01-05
+
+### Improvements
+
+  * `Arrow::Table#save`: Added Parquet save support.
+
+  * `Arrow::Table.load`: Added Parquet load support.
+
 ## 0.0.1 - 2017-05-17
 
 Initial release!!!

data/lib/parquet/arrow-table-loadable.rb

@@ -0,0 +1,30 @@
+# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Parquet
+  module ArrowTableLoadable
+    private
+    def load_as_parquet(path)
+      reader = Parquet::ArrowFileReader.new(path)
+      reader.n_threads = @options[:n_threads] || 4 # TODO
+      reader.read_table
+    end
+  end
+end
+
+module Arrow
+  class TableLoader
+    include Parquet::ArrowTableLoadable
+  end
+end

data/lib/parquet/arrow-table-savable.rb

@@ -0,0 +1,31 @@
+# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Parquet
+  module ArrowTableSavable
+    private
+    def save_as_parquet(path)
+      chunk_size = @options[:chunk_size] || 1024 # TODO
+      Parquet::ArrowFileWriter.open(@table.schema, path) do |writer|
+        writer.write_table(@table, chunk_size)
+      end
+    end
+  end
+end
+
+module Arrow
+  class TableSaver
+    include Parquet::ArrowTableSavable
+  end
+end

data/lib/parquet/loader.rb

@@ -1,4 +1,4 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Copyright 2017-2018 Kouhei Sutou <kou@clear-code.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,6 +21,15 @@ module Parquet
     end
 
     private
+    def post_load(repository, namespace)
+      require_libraries
+    end
+
+    def require_libraries
+      require "parquet/arrow-table-loadable"
+      require "parquet/arrow-table-savable"
+    end
+
     def load_object_info(info)
       super
 

data/lib/parquet/version.rb

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 module Parquet
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/red-parquet.gemspec

@@ -42,7 +42,7 @@ Gem::Specification.new do |spec|
   spec.test_files += Dir.glob("test/**/*")
   spec.extensions = ["dependency-check/Rakefile"]
 
-  spec.add_runtime_dependency("red-arrow")
+  spec.add_runtime_dependency("red-arrow", ">= 0.8.1")
 
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("rake")

data/test/test-arrow-table.rb

@@ -0,0 +1,45 @@
+# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class TestArrowTableReader < Test::Unit::TestCase
+  def setup
+    @count_field = Arrow::Field.new("count", :uint8)
+    @visible_field = Arrow::Field.new("visible", :boolean)
+    schema = Arrow::Schema.new([@count_field, @visible_field])
+    count_arrays = [
+      Arrow::UInt8Array.new([1, 2]),
+      Arrow::UInt8Array.new([4, 8, 16]),
+      Arrow::UInt8Array.new([32, 64]),
+      Arrow::UInt8Array.new([128]),
+    ]
+    visible_arrays = [
+      Arrow::BooleanArray.new([true, false, nil]),
+      Arrow::BooleanArray.new([true]),
+      Arrow::BooleanArray.new([true, false]),
+      Arrow::BooleanArray.new([nil]),
+      Arrow::BooleanArray.new([nil]),
+    ]
+    @count_array = Arrow::ChunkedArray.new(count_arrays)
+    @visible_array = Arrow::ChunkedArray.new(visible_arrays)
+    @count_column = Arrow::Column.new(@count_field, @count_array)
+    @visible_column = Arrow::Column.new(@visible_field, @visible_array)
+    @table = Arrow::Table.new(schema, [@count_column, @visible_column])
+  end
+
+  def test_save_load
+    tempfile = Tempfile.open(["red-parquet", ".parquet"])
+    @table.save(tempfile.path)
+    assert_equal(@table, Arrow::Table.load(tempfile.path))
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-parquet
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-17 00:00:00.000000000 Z
+date: 2018-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -111,12 +111,15 @@ files:
 - doc/text/apache-2.0.txt
 - doc/text/news.md
 - lib/parquet.rb
+- lib/parquet/arrow-table-loadable.rb
+- lib/parquet/arrow-table-savable.rb
 - lib/parquet/loader.rb
 - lib/parquet/version.rb
 - red-parquet.gemspec
 - test/helper.rb
 - test/run-test.rb
 - test/test-arrow-file-reader.rb
+- test/test-arrow-table.rb
 homepage: https://github.com/red-data-tools/red-parquet
 licenses:
 - Apache-2.0
@@ -137,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.5.2.2
 signing_key: 
 specification_version: 4
 summary: Red Parquet is a Ruby bindings of Apache Parquet. Red Parquet is based on
@@ -146,3 +149,4 @@ test_files:
 - test/helper.rb
 - test/run-test.rb
 - test/test-arrow-file-reader.rb
+- test/test-arrow-table.rb