parqueteur 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 056b9208a8bffcd163464dbd2cf276a9b0704e96788b77555d545eb339a4e798
-  data.tar.gz: 1e20d31b1fc6f198fee42546939ce289d71d66f65ffa66562cdd7841e0f24f61
+  metadata.gz: 715ea84521855ea8978e4f944fd6ae07f192bf97fdd40893b4b9bd292a3fe0b5
+  data.tar.gz: d05798d68479c37a8d7028cd65ff35438399de4459d84d4797399940c63513f6
 SHA512:
-  metadata.gz: fe08a7b282c4ededc08acb5aa9f4b485ead828aee4fd1444e8bb1af80cc56ea8c20411aefe136809f91ad808bee52db261218e8b5e6b7538bfa53d1eb38eb4b5
-  data.tar.gz: 0fee8ec94698b7b4c9d3a089fd0094a52bd83dfda56d0652f8a5b08dfe84a88b251736e62a9da7f510e0fa3d1842e2551161178ce30b5e0f5c6ee9b903917a2c
+  metadata.gz: 1a4d74f311c64f79c6e339ba05e631feb94552268936f58c86e0e9bcf70bb46fec8a94d452501cd395203ea33ab9440e015e4503f255cfcecc7703e6fc8d0a1b
+  data.tar.gz: 344ea6420b6c08bbe61f4f534a92c26f563193d3f6dc995dd7eb09d9df5e9f3342971bbd53570d8db0f6b910e457e62e95811c04ba2245578de3b8bb245f7dc1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parqueteur (1.2.0)
+    parqueteur (1.3.0)
       red-parquet (~> 5.0)
 
 GEM

data/README.md

@@ -1,15 +1,29 @@
 # Parqueteur
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/parqueteur`. To experiment with that code, run `bin/console` for an interactive prompt.
+Parqueteur enables you to generate Apache Parquet files from raw data.
 
-TODO: Delete this and the text above, and describe your gem
+## Dependencies
 
+Since I only tested Parqueteur on Ubuntu, I don't have any install scripts for other operating systems.
+### Debian/Ubuntu packages
+- `libgirepository1.0-dev`
+- `libarrow-dev`
+- `libarrow-glib-dev`
+- `libparquet-dev`
+- `libparquet-glib-dev`
+
+You can check `scripts/apache-arrow-ubuntu-install.sh` script for a quick way to install all of them.
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'parqueteur'
+gem 'parqueteur', '~> 1.0'
+```
+
+> (optional) If you don't want to require Parqueteur globally you can add `require: false` to the Gemfile instruction:
+```ruby
+gem 'parqueteur', '~> 1.0', require: false
 ```
 
 And then execute:
@@ -22,14 +36,35 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+Parqueteur provides an elegant way to generate Apache Parquet files from a defined schema.
+```ruby
+require 'parqueteur'
 
-## Development
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+end
 
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+data = [
+  { 'id' => 1, 'reference' => 'hello world 1' },
+  { 'id' => 2, 'reference' => 'hello world 2' },
+  { 'id' => 3, 'reference' => 'hello world 3' }
+]
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('hello_world.parquet')
+
+# in-memory result (StringIO)
+converter.to_io
+
+# write to temporary file (Tempfile)
+# don't forget to `close` / `unlink` it after usage
+converter.to_tmpfile
+```
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/parqueteur.
+Bug reports and pull requests are welcome on GitHub at https://github.com/pocketsizesun/parqueteur-ruby.

data/examples/{convert-with-compression.rb → convert-and-compression.rb}

@@ -3,7 +3,7 @@ require 'parqueteur'
 require 'securerandom'
 require 'benchmark'
 
-class Foo < Parqueteur::Converter
+class FooParquetConverter < Parqueteur::Converter
   column :id, :bigint
   column :reference, :string
   column :hash, :map, key: :string, value: :string
@@ -48,5 +48,9 @@ data = 10000.times.collect do |i|
 end
 puts "data generation OK"
 
-path = 'tmp/test.parquet'
-Foo.convert_to(data, path, compression: :gzip)
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('tmp/example.gzip-compressed.parquet')
+converter.write('tmp/example.no-gzip.parquet', compression: false)

data/examples/hello-world.rb

@@ -0,0 +1,56 @@
+require 'bundler/setup'
+require 'parqueteur'
+
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :my_string_array, :array, elements: :string
+  column :my_date, :date
+  column :my_decimal, :decimal, precision: 12, scale: 4
+  column :my_int, :integer
+  column :my_map, :map, key: :string, value: :string
+  column :my_string, :string
+  column :my_struct, :struct do
+    field :my_struct_str, :string
+    field :my_struct_int, :integer
+  end
+  column :my_time, :time
+  column :my_timestamp, :timestamp
+end
+
+data = 100.times.collect do |i|
+  {
+    'id' => i,
+    'my_string_array' => %w[a b c],
+    'my_date' => Date.today,
+    'my_decimal' => BigDecimal('789000.5678'),
+    'my_int' => rand(1..10),
+    'my_map' => { 'a' => 'b' },
+    'my_string' => 'Hello World',
+    'my_struct' => {
+      'my_struct_str' => 'Hello World',
+      'my_struct_int' => 1
+    },
+    'my_time' => 3600,
+    'my_timestamp' => Time.now
+  }
+end
+
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('tmp/hello_world.compressed.parquet')
+converter.write('tmp/hello_world.parquet', compression: false)
+
+# in-memory result (StringIO)
+converter.to_io
+
+# write to temporary file (Tempfile)
+# don't forget to `close` / `unlink` it after usage
+converter.to_tmpfile
+
+# Arrow Table
+table = converter.to_arrow_table
+table.each_record do |record|
+  pp record.to_h
+end

data/lib/parqueteur/converter.rb

@@ -41,12 +41,14 @@ module Parqueteur
       @compression = kwargs.fetch(:compression, nil)&.to_sym
     end
 
-    def split(size)
+    def split(size, batch_size: nil, compression: nil)
       Enumerator.new do |arr|
+        options = {
+          batch_size: batch_size || @batch_size,
+          compression: compression || @compression
+        }
         @input.each_slice(size) do |records|
-          local_converter = self.class.new(
-            records, batch_size: @batch_size, compression: @compression
-          )
+          local_converter = self.class.new(records, **options)
           file = local_converter.to_tmpfile
           arr << file
           file.close
@@ -55,23 +57,31 @@ module Parqueteur
       end
     end
 
-    def split_by_io(size)
+    def split_by_io(size, batch_size: nil, compression: nil)
       Enumerator.new do |arr|
+        options = {
+          batch_size: batch_size || @batch_size,
+          compression: compression || @compression
+        }
         @input.each_slice(size) do |records|
-          local_converter = self.class.new(records)
+          local_converter = self.class.new(records, **options)
           arr << local_converter.to_io
         end
       end
     end
 
-    def write(path)
+    def write(path, batch_size: nil, compression: nil)
+      compression = @compression if compression.nil?
+      batch_size = @batch_size if batch_size.nil?
       arrow_schema = self.class.columns.arrow_schema
       writer_properties = Parquet::WriterProperties.new
-      writer_properties.set_compression(@compression) unless @compression.nil?
+      if !compression.nil? && compression != false
+        writer_properties.set_compression(compression)
+      end
 
       Arrow::FileOutputStream.open(path, false) do |output|
         Parquet::ArrowFileWriter.open(arrow_schema, output, writer_properties) do |writer|
-          @input.each_slice(@batch_size) do |records|
+          @input.each_slice(batch_size) do |records|
             arrow_table = build_arrow_table(records)
             writer.write_table(arrow_table, 1024)
           end
@@ -81,32 +91,32 @@ module Parqueteur
       true
     end
 
-    def to_tmpfile
+    def to_tmpfile(options = {})
       tempfile = Tempfile.new
       tempfile.binmode
-      write(tempfile.path)
+      write(tempfile.path, **options)
       tempfile.rewind
       tempfile
     end
 
-    def to_io
-      tmpfile = to_tmpfile
+    def to_io(options = {})
+      tmpfile = to_tmpfile(options)
       strio = StringIO.new(tmpfile.read)
       tmpfile.close
       tmpfile.unlink
       strio
     end
 
-    def to_arrow_table
-      file = to_tmpfile
+    def to_arrow_table(options = {})
+      file = to_tmpfile(options)
       table = Arrow::Table.load(file.path, format: :parquet)
       file.close
       file.unlink
       table
     end
 
-    def to_blob
-      to_io.read
+    def to_blob(options = {})
+      to_tmpfile(options).read
     end
 
     private

data/lib/parqueteur/type_resolver.rb

@@ -7,14 +7,24 @@ module Parqueteur
     def self.registered_types
       @registered_types ||= {
         array: Parqueteur::Types::ArrayType,
+        bigdecimal: Parqueteur::Types::Decimal256Type,
         bigint: Parqueteur::Types::Int64Type,
         boolean: Parqueteur::Types::BooleanType,
+        date: Parqueteur::Types::Date32Type,
+        date32: Parqueteur::Types::Date64Type,
+        date64: Parqueteur::Types::Date64Type,
+        decimal: Parqueteur::Types::Decimal128Type,
+        decimal128: Parqueteur::Types::Decimal128Type,
+        decimal256: Parqueteur::Types::Decimal256Type,
         int32: Parqueteur::Types::Int32Type,
         int64: Parqueteur::Types::Int64Type,
         integer: Parqueteur::Types::Int32Type,
         map: Parqueteur::Types::MapType,
         string: Parqueteur::Types::StringType,
         struct: Parqueteur::Types::StructType,
+        time: Parqueteur::Types::Time32Type,
+        time32: Parqueteur::Types::Time32Type,
+        time64: Parqueteur::Types::Time64Type,
         timestamp: Parqueteur::Types::TimestampType
       }
     end

data/lib/parqueteur/types/date32_type.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Date32Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Date32ArrayBuilder.build(values)
+      end
+
+      def arrow_type_builder
+        Arrow::Date32DataType.new
+      end
+    end
+  end
+end

data/lib/parqueteur/types/date64_type.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Date64Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Date64ArrayBuilder.build([values])
+      end
+
+      def arrow_type_builder
+        Arrow::Date64DataType.new
+      end
+    end
+  end
+end

data/lib/parqueteur/types/decimal128_type.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Decimal128Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Decimal128ArrayBuilder.build(@arrow_type, values)
+      end
+
+      def arrow_type_builder
+        Arrow::Decimal128DataType.new(
+          precision: @options.fetch(:precision),
+          scale: @options.fetch(:scale)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/decimal256_type.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Decimal256Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Decimal256ArrayBuilder.build(@arrow_type, values)
+      end
+
+      def arrow_type_builder
+        Arrow::Decimal256DataType.new(
+          precision: @options.fetch(:precision),
+          scale: @options.fetch(:scale)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/int32_type.rb

@@ -21,5 +21,3 @@ module Parqueteur
     end
   end
 end
-
-# when :integer

data/lib/parqueteur/types/int64_type.rb

@@ -21,5 +21,3 @@ module Parqueteur
     end
   end
 end
-
-# when :integer

data/lib/parqueteur/types/time32_type.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Time32Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Time32Array.new(
+          @options.fetch(:precision, :second), values
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::Time32DataType.new(
+          options.fetch(:unit, :second)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/time64_type.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Time64Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Time64Array.new(
+          @options.fetch(:precision, :second), values
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::Time64DataType.new(
+          options.fetch(:unit, :second)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/timestamp_type.rb

@@ -9,7 +9,9 @@ module Parqueteur
   module Types
     class TimestampType < Parqueteur::Type
       def build_value_array(values)
-        Arrow::TimestampArray.new(values)
+        Arrow::TimestampArray.new(
+          options.fetch(:unit, :second), values
+        )
       end
 
       def arrow_type_builder

data/lib/parqueteur/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parqueteur
-  VERSION = '1.2.0'
+  VERSION = '1.3.0'
 end

data/lib/parqueteur.rb

@@ -3,6 +3,7 @@
 require 'json'
 require 'singleton'
 require 'tempfile'
+require 'parquet'
 
 require_relative 'parqueteur/version'
 require 'parqueteur/column'
@@ -14,16 +15,20 @@ require 'parqueteur/type'
 require 'parqueteur/type_resolver'
 require 'parqueteur/types/array_type'
 require 'parqueteur/types/boolean_type'
+require 'parqueteur/types/date32_type'
+require 'parqueteur/types/date64_type'
+require 'parqueteur/types/decimal128_type'
+require 'parqueteur/types/decimal256_type'
 require 'parqueteur/types/int32_type'
 require 'parqueteur/types/int64_type'
 require 'parqueteur/types/map_type'
 require 'parqueteur/types/string_type'
 require 'parqueteur/types/struct_type'
+require 'parqueteur/types/time32_type'
+require 'parqueteur/types/time64_type'
 require 'parqueteur/types/timestamp_type'
-require 'parquet'
 
 module Parqueteur
   class Error < StandardError; end
   class TypeNotFound < Error; end
-  # Your code goes here...
 end

data/parqueteur.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Julien D."]
   spec.email         = ["julien@pocketsizesun.com"]
   spec.license       = 'Apache-2.0'
-  spec.summary       = 'Parqueteur - A Ruby gem that convert JSON to Parquet'
-  spec.description   = 'Convert JSON to Parquet'
+  spec.summary       = 'Parqueteur - A Ruby gem that convert data to Parquet'
+  spec.description   = 'Convert data to Parquet'
   spec.homepage      = 'https://github.com/pocketsizesun/parqueteur-ruby'
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 

data/scripts/apache-arrow-ubuntu-install.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+if [ $(dpkg-query -W -f='${Status}' apache-arrow-apt-source 2>/dev/null | grep -c "ok installed") -eq 1 ]
+then
+  exit 0
+fi
+
+LSB_RELEASE_CODENAME_SHORT=$(lsb_release --codename --short)
+
+apt-get update
+apt-get install -y -V ca-certificates lsb-release wget
+wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+apt-get install -y -V ./apache-arrow-apt-source-latest-${LSB_RELEASE_CODENAME_SHORT}.deb
+rm ./apache-arrow-apt-source-latest-${LSB_RELEASE_CODENAME_SHORT}.deb
+apt-get update
+apt-get install -y libgirepository1.0-dev libarrow-dev libarrow-glib-dev libparquet-dev libparquet-glib-dev
+
+exit 0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parqueteur
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Julien D.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-02 00:00:00.000000000 Z
+date: 2021-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-parquet
@@ -24,7 +24,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-description: Convert JSON to Parquet
+description: Convert data to Parquet
 email:
 - julien@pocketsizesun.com
 executables: []
@@ -38,11 +38,12 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- examples/convert-and-compression.rb
 - examples/convert-methods.rb
 - examples/convert-to-io.rb
 - examples/convert-with-chunks.rb
-- examples/convert-with-compression.rb
 - examples/convert-without-compression.rb
+- examples/hello-world.rb
 - lib/parqueteur.rb
 - lib/parqueteur/column.rb
 - lib/parqueteur/column_collection.rb
@@ -53,15 +54,21 @@ files:
 - lib/parqueteur/type_resolver.rb
 - lib/parqueteur/types/array_type.rb
 - lib/parqueteur/types/boolean_type.rb
+- lib/parqueteur/types/date32_type.rb
+- lib/parqueteur/types/date64_type.rb
+- lib/parqueteur/types/decimal128_type.rb
+- lib/parqueteur/types/decimal256_type.rb
 - lib/parqueteur/types/int32_type.rb
 - lib/parqueteur/types/int64_type.rb
 - lib/parqueteur/types/map_type.rb
 - lib/parqueteur/types/string_type.rb
 - lib/parqueteur/types/struct_type.rb
+- lib/parqueteur/types/time32_type.rb
+- lib/parqueteur/types/time64_type.rb
 - lib/parqueteur/types/timestamp_type.rb
 - lib/parqueteur/version.rb
 - parqueteur.gemspec
-- test.json
+- scripts/apache-arrow-ubuntu-install.sh
 homepage: https://github.com/pocketsizesun/parqueteur-ruby
 licenses:
 - Apache-2.0
@@ -85,5 +92,5 @@ requirements: []
 rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
-summary: Parqueteur - A Ruby gem that convert JSON to Parquet
+summary: Parqueteur - A Ruby gem that convert data to Parquet
 test_files: []

data/test.json

@@ -1 +0,0 @@
-[{"id":1,"reference":"coucou","hash":{"a":"b"},"valid":true,"hash2":{},"numbers":[1,2,3],"map_array":[]},{"id":2,"reference":"coucou","hash":{"c":"d"},"valid":false,"hash2":{},"numbers":[4,5,6],"map_array":[]},{"id":3,"reference":"coucou","hash":{"e":"f"},"valid":true,"hash2":{"x":[1,2,3]},"numbers":[7,8,9],"map_array":[{"x":"y"}]}]