parqueteur 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 715ea84521855ea8978e4f944fd6ae07f192bf97fdd40893b4b9bd292a3fe0b5
-  data.tar.gz: d05798d68479c37a8d7028cd65ff35438399de4459d84d4797399940c63513f6
+  metadata.gz: 86f767a68e38cdd93da015e4ddfcda06b2eefe553ecb6d7a423b4cc0f2752183
+  data.tar.gz: cd741b1d023e44fcc14c08192b9791b675339736527f47dbe2393e85bdff9d07
 SHA512:
-  metadata.gz: 1a4d74f311c64f79c6e339ba05e631feb94552268936f58c86e0e9bcf70bb46fec8a94d452501cd395203ea33ab9440e015e4503f255cfcecc7703e6fc8d0a1b
-  data.tar.gz: 344ea6420b6c08bbe61f4f534a92c26f563193d3f6dc995dd7eb09d9df5e9f3342971bbd53570d8db0f6b910e457e62e95811c04ba2245578de3b8bb245f7dc1
+  metadata.gz: 58882a4d2d1ea5a0cb53f5643f96ac504352b87e6c1436d10841a616d2867bf127c70a31374f6ff2f455fb2894d29fe8cc439ca8a0efca5fe519c00ee312c8c5
+  data.tar.gz: ac185e758d8c0fa19d11ac05e96f77f93cb8260e055d203d1d0d71dd04e0ff6db70d773824a8a6fe5c63e5f235ab179f3417ac689b590403b69192cbad0bde98

data/Gemfile.lock

@@ -1,21 +1,21 @@
 PATH
   remote: .
   specs:
-    parqueteur (1.3.0)
+    parqueteur (1.3.1)
       red-parquet (~> 5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    bigdecimal (3.0.0)
-    extpp (0.0.9)
-    gio2 (3.4.6)
-      gobject-introspection (= 3.4.6)
-    glib2 (3.4.6)
+    bigdecimal (3.0.2)
+    extpp (0.1.0)
+    gio2 (3.4.9)
+      gobject-introspection (= 3.4.9)
+    glib2 (3.4.9)
       native-package-installer (>= 1.0.3)
       pkg-config (>= 1.3.5)
-    gobject-introspection (3.4.6)
-      glib2 (= 3.4.6)
+    gobject-introspection (3.4.9)
+      glib2 (= 3.4.9)
     native-package-installer (1.1.1)
     pkg-config (1.4.6)
     rake (13.0.6)

data/README.md

@@ -1,10 +1,12 @@
 # Parqueteur
 
+[![Gem Version](https://badge.fury.io/rb/parqueteur.svg)](https://badge.fury.io/rb/parqueteur)
+
 Parqueteur enables you to generate Apache Parquet files from raw data.
 
 ## Dependencies
 
-Since I only tested Parqueteur on Ubuntu, I don't have any install scripts for other operating systems.
+Since I only tested Parqueteur on Ubuntu, I don't have any install scripts for others operating systems.
 ### Debian/Ubuntu packages
 - `libgirepository1.0-dev`
 - `libarrow-dev`
@@ -37,18 +39,24 @@ Or install it yourself as:
 ## Usage
 
 Parqueteur provides an elegant way to generate Apache Parquet files from a defined schema.
+
+Converters accepts any object that implements `Enumerable` as data source.
+
+### Working example
+
 ```ruby
 require 'parqueteur'
 
 class FooParquetConverter < Parqueteur::Converter
   column :id, :bigint
   column :reference, :string
+  column :datetime, :timestamp
 end
 
 data = [
-  { 'id' => 1, 'reference' => 'hello world 1' },
-  { 'id' => 2, 'reference' => 'hello world 2' },
-  { 'id' => 3, 'reference' => 'hello world 3' }
+  { 'id' => 1, 'reference' => 'hello world 1', 'datetime' => Time.now },
+  { 'id' => 2, 'reference' => 'hello world 2', 'datetime' => Time.now },
+  { 'id' => 3, 'reference' => 'hello world 3', 'datetime' => Time.now }
 ]
 
 # initialize Converter with Parquet GZIP compression mode
@@ -63,8 +71,94 @@ converter.to_io
 # write to temporary file (Tempfile)
 # don't forget to `close` / `unlink` it after usage
 converter.to_tmpfile
+
+# convert to Arrow::Table
+pp converter.to_arrow_table
+```
+
+### Using transformers
+
+You can use transformers to apply data items transformations.
+
+From `examples/cars.rb`:
+
+```ruby
+require 'parqueteur'
+
+class Car
+  attr_reader :name, :production_year
+
+  def initialize(name, production_year)
+    @name = name
+    @production_year = production_year
+  end
+end
+
+class CarParquetConverter < Parqueteur::Converter
+  column :name, :string
+  column :production_year, :integer
+
+  transform do |car|
+    {
+      'name' => car.name,
+      'production_year' => car.production_year
+    }
+  end
+end
+
+cars = [
+  Car.new('Alfa Romeo 75', 1985),
+  Car.new('Alfa Romeo 33', 1983),
+  Car.new('Audi A3', 1996),
+  Car.new('Audi A4', 1994),
+  Car.new('BMW 503', 1956),
+  Car.new('BMW X5', 1999)
+]
+
+# initialize Converter with Parquet GZIP compression mode
+converter = CarParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+pp converter.to_arrow_table
 ```
 
+Output:
+```
+#<Arrow::Table:0x7fc1fb24b958 ptr=0x7fc1faedd910>
+#     name           production_year
+0     Alfa Romeo 75  1985
+1     Alfa Romeo 33  1983
+2     Audi A3        1996
+3     Audi A4        1994
+4     BMW 503        1956
+5     BMW X5         1999
+```
+
+### Available Types
+
+| Name (Symbol) | Apache Parquet Type |
+| ------------- | --------- |
+| `:array` | `Array` |
+| `:bigdecimal` | `Decimal256` |
+| `:bigint` | `Int64` or `UInt64` with `unsigned: true` option |
+| `:boolean` | `Boolean` |
+| `:date` | `Date32` |
+| `:date32` | `Date32` |
+| `:date64` | `Date64` |
+| `:decimal` | `Decimal128` |
+| `:decimal128` | `Decimal128` |
+| `:decimal256` | `Decimal256` |
+| `:int32` | `Int32` or `UInt32` with `unsigned: true` option |
+| `:int64` | `Int64` or `UInt64` with `unsigned: true` option |
+| `:integer` | `Int32` or `UInt32` with `unsigned: true` option |
+| `:map` | `Map` |
+| `:string` | `String` |
+| `:struct` | `Struct` |
+| `:time` | `Time32` |
+| `:time32` | `Time32` |
+| `:time64` | `Time64` |
+| `:timestamp` | `Timestamp` |
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/pocketsizesun/parqueteur-ruby.

data/examples/cars.rb

@@ -0,0 +1,40 @@
+require 'bundler/setup'
+require 'parqueteur'
+
+class Car
+  attr_reader :name, :production_year
+
+  def initialize(name, production_year)
+    @name = name
+    @production_year = production_year
+  end
+end
+
+class CarParquetConverter < Parqueteur::Converter
+  column :name, :string
+  column :production_year, :integer
+
+  transform do |car|
+    {
+      'name' => car.name,
+      'production_year' => car.production_year
+    }
+  end
+end
+
+cars = [
+  Car.new('Alfa Romeo 75', 1985),
+  Car.new('Alfa Romeo 33', 1983),
+  Car.new('Audi A3', 1996),
+  Car.new('Audi A4', 1994),
+  Car.new('BMW 503', 1956),
+  Car.new('BMW X5', 1999)
+]
+
+# initialize Converter with Parquet GZIP compression mode
+converter = CarParquetConverter.new(
+  cars, compression: :gzip
+)
+
+# write result to file
+pp converter.to_arrow_table

data/examples/hello-world.rb

@@ -22,7 +22,7 @@ data = 100.times.collect do |i|
     'id' => i,
     'my_string_array' => %w[a b c],
     'my_date' => Date.today,
-    'my_decimal' => BigDecimal('789000.5678'),
+    'my_decimal' => BigDecimal('0.03'),
     'my_int' => rand(1..10),
     'my_map' => { 'a' => 'b' },
     'my_string' => 'Hello World',
@@ -52,5 +52,6 @@ converter.to_tmpfile
 # Arrow Table
 table = converter.to_arrow_table
 table.each_record do |record|
+  # pp record['my_decimal'].to_f
   pp record.to_h
 end

data/examples/readme-example.rb

@@ -0,0 +1,44 @@
+require 'bundler/setup'
+require 'parqueteur'
+
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :datetime, :timestamp
+  column :beers_count, :integer
+
+  transform do |item|
+    item.merge(
+      'datetime' => Time.now
+    )
+  end
+
+  transform :add_beers
+
+  private
+
+  def add_beers(item)
+    item['beers_count'] += rand(1..3)
+    item
+  end
+end
+
+data = 10.times.lazy.map do |i|
+  { 'id' => i + 1, 'reference' => 'hello world 1', 'beers_count' => 0 }
+end
+
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('tmp/hello_world.parquet')
+
+# in-memory result (StringIO)
+converter.to_io
+
+# write to temporary file (Tempfile)
+# don't forget to `close` / `unlink` it after usage
+converter.to_tmpfile
+
+# convert to Arrow::Table
+pp converter.to_arrow_table

data/lib/parqueteur/type_resolver.rb

@@ -11,7 +11,7 @@ module Parqueteur
         bigint: Parqueteur::Types::Int64Type,
         boolean: Parqueteur::Types::BooleanType,
         date: Parqueteur::Types::Date32Type,
-        date32: Parqueteur::Types::Date64Type,
+        date32: Parqueteur::Types::Date32Type,
         date64: Parqueteur::Types::Date64Type,
         decimal: Parqueteur::Types::Decimal128Type,
         decimal128: Parqueteur::Types::Decimal128Type,

data/lib/parqueteur/types/decimal128_type.rb

@@ -3,14 +3,25 @@
 module Parqueteur
   module Types
     class Decimal128Type < Parqueteur::Type
+      def initialize(options = {}, &block)
+        @scale = options.fetch(:scale)
+        @precision = options.fetch(:precision)
+        @format_str = "%.#{@scale}f"
+        super(options, &block)
+      end
+
       def build_value_array(values)
-        Arrow::Decimal128ArrayBuilder.build(@arrow_type, values)
+        Arrow::Decimal128ArrayBuilder.build(
+          @arrow_type,
+          values.map do |value|
+            Arrow::Decimal128.new(format(@format_str, BigDecimal(value)))
+          end
+        )
       end
 
       def arrow_type_builder
         Arrow::Decimal128DataType.new(
-          precision: @options.fetch(:precision),
-          scale: @options.fetch(:scale)
+          @precision, @scale
         )
       end
     end

data/lib/parqueteur/types/decimal256_type.rb

@@ -3,14 +3,25 @@
 module Parqueteur
   module Types
     class Decimal256Type < Parqueteur::Type
+      def initialize(options = {}, &block)
+        @scale = options.fetch(:scale)
+        @precision = options.fetch(:precision)
+        @format_str = "%.#{@scale}f"
+        super(options, &block)
+      end
+
       def build_value_array(values)
-        Arrow::Decimal256ArrayBuilder.build(@arrow_type, values)
+        Arrow::Decimal256ArrayBuilder.build(
+          @arrow_type,
+          values.map do |value|
+            Arrow::Decimal256.new(format(@format_str, BigDecimal(value)))
+          end
+        )
       end
 
       def arrow_type_builder
         Arrow::Decimal256DataType.new(
-          precision: @options.fetch(:precision),
-          scale: @options.fetch(:scale)
+          @precision, @scale
         )
       end
     end

data/lib/parqueteur/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parqueteur
-  VERSION = '1.3.0'
+  VERSION = '1.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parqueteur
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Julien D.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-03 00:00:00.000000000 Z
+date: 2021-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-parquet
@@ -38,12 +38,14 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- examples/cars.rb
 - examples/convert-and-compression.rb
 - examples/convert-methods.rb
 - examples/convert-to-io.rb
 - examples/convert-with-chunks.rb
 - examples/convert-without-compression.rb
 - examples/hello-world.rb
+- examples/readme-example.rb
 - lib/parqueteur.rb
 - lib/parqueteur/column.rb
 - lib/parqueteur/column_collection.rb