parqueteur 1.1.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90ffcadf5b78e4ffc3329eac9be1be34af9209a77a39a6395eefc1c56afa7ce6
-  data.tar.gz: 1d7f5257d3f86443e0d13b789d7449565198f6cda1563d111a36ad3728264044
+  metadata.gz: 86f767a68e38cdd93da015e4ddfcda06b2eefe553ecb6d7a423b4cc0f2752183
+  data.tar.gz: cd741b1d023e44fcc14c08192b9791b675339736527f47dbe2393e85bdff9d07
 SHA512:
-  metadata.gz: 262039094dd3aa5890f9d1a87d836eb64004c51fc57456d7880a36332295ac8a447fcd045c6bd2e053986ad4e9981021448b39e20010d0e41fef6b7e233d91ca
-  data.tar.gz: 73988e1f836acbe22e26c20b8559d8f79e9eeda55f9612fbefae7aa5bef131187b7a0716a3bd7804db5321b29b19f54b383e8594830814ae765d2c7b30986f59
+  metadata.gz: 58882a4d2d1ea5a0cb53f5643f96ac504352b87e6c1436d10841a616d2867bf127c70a31374f6ff2f455fb2894d29fe8cc439ca8a0efca5fe519c00ee312c8c5
+  data.tar.gz: ac185e758d8c0fa19d11ac05e96f77f93cb8260e055d203d1d0d71dd04e0ff6db70d773824a8a6fe5c63e5f235ab179f3417ac689b590403b69192cbad0bde98

data/Gemfile.lock

@@ -1,21 +1,21 @@
 PATH
   remote: .
   specs:
-    parqueteur (1.1.0)
+    parqueteur (1.3.1)
       red-parquet (~> 5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    bigdecimal (3.0.0)
-    extpp (0.0.9)
-    gio2 (3.4.6)
-      gobject-introspection (= 3.4.6)
-    glib2 (3.4.6)
+    bigdecimal (3.0.2)
+    extpp (0.1.0)
+    gio2 (3.4.9)
+      gobject-introspection (= 3.4.9)
+    glib2 (3.4.9)
       native-package-installer (>= 1.0.3)
       pkg-config (>= 1.3.5)
-    gobject-introspection (3.4.6)
-      glib2 (= 3.4.6)
+    gobject-introspection (3.4.9)
+      glib2 (= 3.4.9)
     native-package-installer (1.1.1)
     pkg-config (1.4.6)
     rake (13.0.6)

data/README.md

@@ -1,15 +1,31 @@
 # Parqueteur
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/parqueteur`. To experiment with that code, run `bin/console` for an interactive prompt.
+[![Gem Version](https://badge.fury.io/rb/parqueteur.svg)](https://badge.fury.io/rb/parqueteur)
 
-TODO: Delete this and the text above, and describe your gem
+Parqueteur enables you to generate Apache Parquet files from raw data.
 
+## Dependencies
+
+Since I only tested Parqueteur on Ubuntu, I don't have any install scripts for others operating systems.
+### Debian/Ubuntu packages
+- `libgirepository1.0-dev`
+- `libarrow-dev`
+- `libarrow-glib-dev`
+- `libparquet-dev`
+- `libparquet-glib-dev`
+
+You can check `scripts/apache-arrow-ubuntu-install.sh` script for a quick way to install all of them.
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'parqueteur'
+gem 'parqueteur', '~> 1.0'
+```
+
+> (optional) If you don't want to require Parqueteur globally you can add `require: false` to the Gemfile instruction:
+```ruby
+gem 'parqueteur', '~> 1.0', require: false
 ```
 
 And then execute:
@@ -22,14 +38,127 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+Parqueteur provides an elegant way to generate Apache Parquet files from a defined schema.
+
+Converters accepts any object that implements `Enumerable` as data source.
+
+### Working example
+
+```ruby
+require 'parqueteur'
+
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :datetime, :timestamp
+end
+
+data = [
+  { 'id' => 1, 'reference' => 'hello world 1', 'datetime' => Time.now },
+  { 'id' => 2, 'reference' => 'hello world 2', 'datetime' => Time.now },
+  { 'id' => 3, 'reference' => 'hello world 3', 'datetime' => Time.now }
+]
+
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
 
-## Development
+# write result to file
+converter.write('hello_world.parquet')
 
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+# in-memory result (StringIO)
+converter.to_io
+
+# write to temporary file (Tempfile)
+# don't forget to `close` / `unlink` it after usage
+converter.to_tmpfile
+
+# convert to Arrow::Table
+pp converter.to_arrow_table
+```
+
+### Using transformers
+
+You can use transformers to apply data items transformations.
+
+From `examples/cars.rb`:
+
+```ruby
+require 'parqueteur'
+
+class Car
+  attr_reader :name, :production_year
+
+  def initialize(name, production_year)
+    @name = name
+    @production_year = production_year
+  end
+end
+
+class CarParquetConverter < Parqueteur::Converter
+  column :name, :string
+  column :production_year, :integer
+
+  transform do |car|
+    {
+      'name' => car.name,
+      'production_year' => car.production_year
+    }
+  end
+end
+
+cars = [
+  Car.new('Alfa Romeo 75', 1985),
+  Car.new('Alfa Romeo 33', 1983),
+  Car.new('Audi A3', 1996),
+  Car.new('Audi A4', 1994),
+  Car.new('BMW 503', 1956),
+  Car.new('BMW X5', 1999)
+]
+
+# initialize Converter with Parquet GZIP compression mode
+converter = CarParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+pp converter.to_arrow_table
+```
+
+Output:
+```
+#<Arrow::Table:0x7fc1fb24b958 ptr=0x7fc1faedd910>
+#     name           production_year
+0     Alfa Romeo 75  1985
+1     Alfa Romeo 33  1983
+2     Audi A3        1996
+3     Audi A4        1994
+4     BMW 503        1956
+5     BMW X5         1999
+```
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+### Available Types
+
+| Name (Symbol) | Apache Parquet Type |
+| ------------- | --------- |
+| `:array` | `Array` |
+| `:bigdecimal` | `Decimal256` |
+| `:bigint` | `Int64` or `UInt64` with `unsigned: true` option |
+| `:boolean` | `Boolean` |
+| `:date` | `Date32` |
+| `:date32` | `Date32` |
+| `:date64` | `Date64` |
+| `:decimal` | `Decimal128` |
+| `:decimal128` | `Decimal128` |
+| `:decimal256` | `Decimal256` |
+| `:int32` | `Int32` or `UInt32` with `unsigned: true` option |
+| `:int64` | `Int64` or `UInt64` with `unsigned: true` option |
+| `:integer` | `Int32` or `UInt32` with `unsigned: true` option |
+| `:map` | `Map` |
+| `:string` | `String` |
+| `:struct` | `Struct` |
+| `:time` | `Time32` |
+| `:time32` | `Time32` |
+| `:time64` | `Time64` |
+| `:timestamp` | `Timestamp` |
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/parqueteur.
+Bug reports and pull requests are welcome on GitHub at https://github.com/pocketsizesun/parqueteur-ruby.

data/examples/cars.rb

@@ -0,0 +1,40 @@
+require 'bundler/setup'
+require 'parqueteur'
+
+class Car
+  attr_reader :name, :production_year
+
+  def initialize(name, production_year)
+    @name = name
+    @production_year = production_year
+  end
+end
+
+class CarParquetConverter < Parqueteur::Converter
+  column :name, :string
+  column :production_year, :integer
+
+  transform do |car|
+    {
+      'name' => car.name,
+      'production_year' => car.production_year
+    }
+  end
+end
+
+cars = [
+  Car.new('Alfa Romeo 75', 1985),
+  Car.new('Alfa Romeo 33', 1983),
+  Car.new('Audi A3', 1996),
+  Car.new('Audi A4', 1994),
+  Car.new('BMW 503', 1956),
+  Car.new('BMW X5', 1999)
+]
+
+# initialize Converter with Parquet GZIP compression mode
+converter = CarParquetConverter.new(
+  cars, compression: :gzip
+)
+
+# write result to file
+pp converter.to_arrow_table

data/examples/convert-and-compression.rb

@@ -0,0 +1,56 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('tmp/example.gzip-compressed.parquet')
+converter.write('tmp/example.no-gzip.parquet', compression: false)

data/examples/convert-methods.rb

@@ -0,0 +1,54 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+converter = Foo.new(data, compression: :gzip)
+pp converter.to_io
+pp converter.to_arrow_table
+converter.write('tmp/test.parquet')

data/examples/convert-to-io.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+io = Foo.convert(data)
+pp io.read

data/examples/convert-with-chunks.rb

@@ -0,0 +1,54 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+converter = Foo.new(data, compression: :gzip)
+converter.split(200).each_with_index do |chunk, idx|
+  puts "#{idx}: #{chunk.path}"
+end

data/examples/convert-without-compression.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+path = 'tmp/test.parquet'
+Foo.convert_to(data, path)

data/examples/hello-world.rb

@@ -0,0 +1,57 @@
+require 'bundler/setup'
+require 'parqueteur'
+
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :my_string_array, :array, elements: :string
+  column :my_date, :date
+  column :my_decimal, :decimal, precision: 12, scale: 4
+  column :my_int, :integer
+  column :my_map, :map, key: :string, value: :string
+  column :my_string, :string
+  column :my_struct, :struct do
+    field :my_struct_str, :string
+    field :my_struct_int, :integer
+  end
+  column :my_time, :time
+  column :my_timestamp, :timestamp
+end
+
+data = 100.times.collect do |i|
+  {
+    'id' => i,
+    'my_string_array' => %w[a b c],
+    'my_date' => Date.today,
+    'my_decimal' => BigDecimal('0.03'),
+    'my_int' => rand(1..10),
+    'my_map' => { 'a' => 'b' },
+    'my_string' => 'Hello World',
+    'my_struct' => {
+      'my_struct_str' => 'Hello World',
+      'my_struct_int' => 1
+    },
+    'my_time' => 3600,
+    'my_timestamp' => Time.now
+  }
+end
+
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('tmp/hello_world.compressed.parquet')
+converter.write('tmp/hello_world.parquet', compression: false)
+
+# in-memory result (StringIO)
+converter.to_io
+
+# write to temporary file (Tempfile)
+# don't forget to `close` / `unlink` it after usage
+converter.to_tmpfile
+
+# Arrow Table
+table = converter.to_arrow_table
+table.each_record do |record|
+  # pp record['my_decimal'].to_f
+  pp record.to_h
+end

data/examples/readme-example.rb

@@ -0,0 +1,44 @@
+require 'bundler/setup'
+require 'parqueteur'
+
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :datetime, :timestamp
+  column :beers_count, :integer
+
+  transform do |item|
+    item.merge(
+      'datetime' => Time.now
+    )
+  end
+
+  transform :add_beers
+
+  private
+
+  def add_beers(item)
+    item['beers_count'] += rand(1..3)
+    item
+  end
+end
+
+data = 10.times.lazy.map do |i|
+  { 'id' => i + 1, 'reference' => 'hello world 1', 'beers_count' => 0 }
+end
+
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('tmp/hello_world.parquet')
+
+# in-memory result (StringIO)
+converter.to_io
+
+# write to temporary file (Tempfile)
+# don't forget to `close` / `unlink` it after usage
+converter.to_tmpfile
+
+# convert to Arrow::Table
+pp converter.to_arrow_table

data/lib/parqueteur/converter.rb

@@ -2,7 +2,7 @@
 
 module Parqueteur
   class Converter
-    attr_reader :schema
+    DEFAULT_BATCH_SIZE = 10
 
     def self.inline(&block)
       Class.new(self, &block)
@@ -24,104 +24,137 @@ module Parqueteur
       transforms << (method_name || block)
     end
 
-    def self.convert(input, output: nil)
-      converter = new(input)
-      if !output.nil?
-        converter.write(output)
-      else
-        converter.to_blob
+    def self.convert(input, **kwargs)
+      new(input, **kwargs).to_io
+    end
+
+    def self.convert_to(input, output_path, **kwargs)
+      converter = new(input, **kwargs)
+      converter.write(output_path)
+    end
+
+    # @param [Enumerable] An enumerable object
+    # @option [Symbol] compression - :gzip
+    def initialize(input, **kwargs)
+      @input = Parqueteur::Input.from(input)
+      @batch_size = kwargs.fetch(:batch_size, DEFAULT_BATCH_SIZE)
+      @compression = kwargs.fetch(:compression, nil)&.to_sym
+    end
+
+    def split(size, batch_size: nil, compression: nil)
+      Enumerator.new do |arr|
+        options = {
+          batch_size: batch_size || @batch_size,
+          compression: compression || @compression
+        }
+        @input.each_slice(size) do |records|
+          local_converter = self.class.new(records, **options)
+          file = local_converter.to_tmpfile
+          arr << file
+          file.close
+          file.unlink
+        end
       end
     end
 
-    def initialize(input, options = {})
-      @input = Parqueteur::Input.from(input, options)
-    end
-
-    def write(output)
-      case output
-      when :io
-        to_io
-      when String
-        to_arrow_table.save(output)
-      when StringIO, IO
-        buffer = Arrow::ResizableBuffer.new(0)
-        to_arrow_table.save(buffer, format: :parquet)
-        output.write(buffer.data.to_s)
-        output.rewind
-        output
-      else
-        raise ArgumentError, "unsupported output: #{output.class}, accepted: String (filename), IO, StringIO"
+    def split_by_io(size, batch_size: nil, compression: nil)
+      Enumerator.new do |arr|
+        options = {
+          batch_size: batch_size || @batch_size,
+          compression: compression || @compression
+        }
+        @input.each_slice(size) do |records|
+          local_converter = self.class.new(records, **options)
+          arr << local_converter.to_io
+        end
+      end
+    end
+
+    def write(path, batch_size: nil, compression: nil)
+      compression = @compression if compression.nil?
+      batch_size = @batch_size if batch_size.nil?
+      arrow_schema = self.class.columns.arrow_schema
+      writer_properties = Parquet::WriterProperties.new
+      if !compression.nil? && compression != false
+        writer_properties.set_compression(compression)
+      end
+
+      Arrow::FileOutputStream.open(path, false) do |output|
+        Parquet::ArrowFileWriter.open(arrow_schema, output, writer_properties) do |writer|
+          @input.each_slice(batch_size) do |records|
+            arrow_table = build_arrow_table(records)
+            writer.write_table(arrow_table, 1024)
+          end
+        end
       end
+
+      true
+    end
+
+    def to_tmpfile(options = {})
+      tempfile = Tempfile.new
+      tempfile.binmode
+      write(tempfile.path, **options)
+      tempfile.rewind
+      tempfile
     end
 
-    def to_s
-      inspect
+    def to_io(options = {})
+      tmpfile = to_tmpfile(options)
+      strio = StringIO.new(tmpfile.read)
+      tmpfile.close
+      tmpfile.unlink
+      strio
     end
 
-    def to_io
-      write(StringIO.new)
+    def to_arrow_table(options = {})
+      file = to_tmpfile(options)
+      table = Arrow::Table.load(file.path, format: :parquet)
+      file.close
+      file.unlink
+      table
     end
 
-    def to_blob
-      write(StringIO.new).read
+    def to_blob(options = {})
+      to_tmpfile(options).read
     end
 
-    def to_arrow_table
+    private
+
+    def build_arrow_table(records)
       transforms = self.class.transforms
 
-      chunks = self.class.columns.each_with_object({}) do |column, hash|
+      values = self.class.columns.each_with_object({}) do |column, hash|
         hash[column.name] = []
       end
-      items_count = 0
-      @input.each_slice(100) do |items|
-        values = self.class.columns.each_with_object({}) do |column, hash|
-          hash[column.name] = []
-        end
 
-        items.each do |item|
-          if transforms.length > 0
-            transforms.each do |transform|
-              item = \
-                if transform.is_a?(Symbol)
-                  __send__(transform, item)
-                else
-                  transform.call(item)
-                end
-            end
+      records.each do |item|
+        if transforms.length > 0
+          transforms.each do |transform|
+            item = \
+              if transform.is_a?(Symbol)
+                __send__(transform, item)
+              else
+                transform.call(item)
+              end
           end
+        end
 
-          values.each_key do |value_key|
-            if item.key?(value_key)
-              values[value_key] << item[value_key]
-            else
-              values[value_key] << nil
-            end
+        values.each_key do |value_key|
+          if item.key?(value_key)
+            values[value_key] << item[value_key]
+          else
+            values[value_key] << nil
           end
         end
+      end
 
-        values.each_with_object(chunks) do |item, hash|
+      Arrow::Table.new(
+        values.each_with_object({}) do |item, hash|
           column = self.class.columns.find(item[0])
-          hash[item[0]].push(
-            column.type.build_value_array(item[1])
-          )
+          hash[item[0]] = column.type.build_value_array(item[1])
         end
-
-        items_count += items.length
-      end
-
-      if items_count > 0
-        Arrow::Table.new(
-          chunks.transform_values! do |value|
-            Arrow::ChunkedArray.new(value)
-          end
-        )
-      else
-        Arrow::Table.new(
-          self.class.columns.each_with_object({}) do |column, hash|
-            hash[column.name] = column.type.build_value_array([])
-          end
-        )
-      end
+      )
     end
   end
 end

data/lib/parqueteur/input.rb

@@ -4,40 +4,25 @@ module Parqueteur
   class Input
     include Enumerable
 
-    def self.from(arg, options = {})
-      new(
-        case arg
-        when String
-          if File.exist?(arg)
-            File.new(arg, 'r')
-          else
-            arg.split("\n")
-          end
-        when Enumerable
-          arg
-        end,
-        options
-      )
+    def self.from(arg)
+      return arg if arg.is_a?(self)
+
+      new(arg)
     end
 
-    def initialize(source, options = {})
+    def initialize(source)
+      unless source.is_a?(Enumerable)
+        raise ArgumentError, 'Enumerable object expected'
+      end
+
       @source = source
-      @options = options
     end
 
     def each(&block)
-      case @source
-      when File
-        if @options.fetch(:json_newlines, true) == true
-          @source.each_line do |line|
-            yield(JSON.parse(line.strip))
-          end
-        else
-          JSON.parse(@source.read).each(&block)
-        end
-        @source.rewind
-      when Enumerable
+      if block_given?
         @source.each(&block)
+      else
+        @source.to_enum(:each)
       end
     end
   end

data/lib/parqueteur/type_resolver.rb

@@ -7,14 +7,24 @@ module Parqueteur
     def self.registered_types
       @registered_types ||= {
         array: Parqueteur::Types::ArrayType,
+        bigdecimal: Parqueteur::Types::Decimal256Type,
         bigint: Parqueteur::Types::Int64Type,
         boolean: Parqueteur::Types::BooleanType,
+        date: Parqueteur::Types::Date32Type,
+        date32: Parqueteur::Types::Date32Type,
+        date64: Parqueteur::Types::Date64Type,
+        decimal: Parqueteur::Types::Decimal128Type,
+        decimal128: Parqueteur::Types::Decimal128Type,
+        decimal256: Parqueteur::Types::Decimal256Type,
         int32: Parqueteur::Types::Int32Type,
         int64: Parqueteur::Types::Int64Type,
         integer: Parqueteur::Types::Int32Type,
         map: Parqueteur::Types::MapType,
         string: Parqueteur::Types::StringType,
         struct: Parqueteur::Types::StructType,
+        time: Parqueteur::Types::Time32Type,
+        time32: Parqueteur::Types::Time32Type,
+        time64: Parqueteur::Types::Time64Type,
         timestamp: Parqueteur::Types::TimestampType
       }
     end

data/lib/parqueteur/types/date32_type.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Date32Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Date32ArrayBuilder.build(values)
+      end
+
+      def arrow_type_builder
+        Arrow::Date32DataType.new
+      end
+    end
+  end
+end

data/lib/parqueteur/types/date64_type.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Date64Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Date64ArrayBuilder.build([values])
+      end
+
+      def arrow_type_builder
+        Arrow::Date64DataType.new
+      end
+    end
+  end
+end

data/lib/parqueteur/types/decimal128_type.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Decimal128Type < Parqueteur::Type
+      def initialize(options = {}, &block)
+        @scale = options.fetch(:scale)
+        @precision = options.fetch(:precision)
+        @format_str = "%.#{@scale}f"
+        super(options, &block)
+      end
+
+      def build_value_array(values)
+        Arrow::Decimal128ArrayBuilder.build(
+          @arrow_type,
+          values.map do |value|
+            Arrow::Decimal128.new(format(@format_str, BigDecimal(value)))
+          end
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::Decimal128DataType.new(
+          @precision, @scale
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/decimal256_type.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Decimal256Type < Parqueteur::Type
+      def initialize(options = {}, &block)
+        @scale = options.fetch(:scale)
+        @precision = options.fetch(:precision)
+        @format_str = "%.#{@scale}f"
+        super(options, &block)
+      end
+
+      def build_value_array(values)
+        Arrow::Decimal256ArrayBuilder.build(
+          @arrow_type,
+          values.map do |value|
+            Arrow::Decimal256.new(format(@format_str, BigDecimal(value)))
+          end
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::Decimal256DataType.new(
+          @precision, @scale
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/int32_type.rb

@@ -21,5 +21,3 @@ module Parqueteur
     end
   end
 end
-
-# when :integer

data/lib/parqueteur/types/int64_type.rb

@@ -21,5 +21,3 @@ module Parqueteur
     end
   end
 end
-
-# when :integer

data/lib/parqueteur/types/time32_type.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Time32Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Time32Array.new(
+          @options.fetch(:precision, :second), values
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::Time32DataType.new(
+          options.fetch(:unit, :second)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/time64_type.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Time64Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Time64Array.new(
+          @options.fetch(:precision, :second), values
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::Time64DataType.new(
+          options.fetch(:unit, :second)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/timestamp_type.rb

@@ -9,7 +9,9 @@ module Parqueteur
   module Types
     class TimestampType < Parqueteur::Type
       def build_value_array(values)
-        Arrow::TimestampArray.new(values)
+        Arrow::TimestampArray.new(
+          options.fetch(:unit, :second), values
+        )
       end
 
       def arrow_type_builder

data/lib/parqueteur/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parqueteur
-  VERSION = '1.1.0'
+  VERSION = '1.3.1'
 end

data/lib/parqueteur.rb

@@ -2,9 +2,10 @@
 
 require 'json'
 require 'singleton'
+require 'tempfile'
+require 'parquet'
 
-require_relative "parqueteur/version"
-require 'parqueteur/chunked_converter'
+require_relative 'parqueteur/version'
 require 'parqueteur/column'
 require 'parqueteur/column_collection'
 require 'parqueteur/converter'
@@ -14,16 +15,20 @@ require 'parqueteur/type'
 require 'parqueteur/type_resolver'
 require 'parqueteur/types/array_type'
 require 'parqueteur/types/boolean_type'
+require 'parqueteur/types/date32_type'
+require 'parqueteur/types/date64_type'
+require 'parqueteur/types/decimal128_type'
+require 'parqueteur/types/decimal256_type'
 require 'parqueteur/types/int32_type'
 require 'parqueteur/types/int64_type'
 require 'parqueteur/types/map_type'
 require 'parqueteur/types/string_type'
 require 'parqueteur/types/struct_type'
+require 'parqueteur/types/time32_type'
+require 'parqueteur/types/time64_type'
 require 'parqueteur/types/timestamp_type'
-require 'parquet'
 
 module Parqueteur
   class Error < StandardError; end
   class TypeNotFound < Error; end
-  # Your code goes here...
 end

data/parqueteur.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Julien D."]
   spec.email         = ["julien@pocketsizesun.com"]
   spec.license       = 'Apache-2.0'
-  spec.summary       = 'Parqueteur - A Ruby gem that convert JSON to Parquet'
-  spec.description   = 'Convert JSON to Parquet'
+  spec.summary       = 'Parqueteur - A Ruby gem that convert data to Parquet'
+  spec.description   = 'Convert data to Parquet'
   spec.homepage      = 'https://github.com/pocketsizesun/parqueteur-ruby'
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 

data/scripts/apache-arrow-ubuntu-install.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+if [ $(dpkg-query -W -f='${Status}' apache-arrow-apt-source 2>/dev/null | grep -c "ok installed") -eq 1 ]
+then
+  exit 0
+fi
+
+LSB_RELEASE_CODENAME_SHORT=$(lsb_release --codename --short)
+
+apt-get update
+apt-get install -y -V ca-certificates lsb-release wget
+wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+apt-get install -y -V ./apache-arrow-apt-source-latest-${LSB_RELEASE_CODENAME_SHORT}.deb
+rm ./apache-arrow-apt-source-latest-${LSB_RELEASE_CODENAME_SHORT}.deb
+apt-get update
+apt-get install -y libgirepository1.0-dev libarrow-dev libarrow-glib-dev libparquet-dev libparquet-glib-dev
+
+exit 0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parqueteur
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Julien D.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-02 00:00:00.000000000 Z
+date: 2021-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-parquet
@@ -24,7 +24,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-description: Convert JSON to Parquet
+description: Convert data to Parquet
 email:
 - julien@pocketsizesun.com
 executables: []
@@ -38,9 +38,15 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- example.rb
+- examples/cars.rb
+- examples/convert-and-compression.rb
+- examples/convert-methods.rb
+- examples/convert-to-io.rb
+- examples/convert-with-chunks.rb
+- examples/convert-without-compression.rb
+- examples/hello-world.rb
+- examples/readme-example.rb
 - lib/parqueteur.rb
-- lib/parqueteur/chunked_converter.rb
 - lib/parqueteur/column.rb
 - lib/parqueteur/column_collection.rb
 - lib/parqueteur/converter.rb
@@ -50,15 +56,21 @@ files:
 - lib/parqueteur/type_resolver.rb
 - lib/parqueteur/types/array_type.rb
 - lib/parqueteur/types/boolean_type.rb
+- lib/parqueteur/types/date32_type.rb
+- lib/parqueteur/types/date64_type.rb
+- lib/parqueteur/types/decimal128_type.rb
+- lib/parqueteur/types/decimal256_type.rb
 - lib/parqueteur/types/int32_type.rb
 - lib/parqueteur/types/int64_type.rb
 - lib/parqueteur/types/map_type.rb
 - lib/parqueteur/types/string_type.rb
 - lib/parqueteur/types/struct_type.rb
+- lib/parqueteur/types/time32_type.rb
+- lib/parqueteur/types/time64_type.rb
 - lib/parqueteur/types/timestamp_type.rb
 - lib/parqueteur/version.rb
 - parqueteur.gemspec
-- test.json
+- scripts/apache-arrow-ubuntu-install.sh
 homepage: https://github.com/pocketsizesun/parqueteur-ruby
 licenses:
 - Apache-2.0
@@ -82,5 +94,5 @@ requirements: []
 rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
-summary: Parqueteur - A Ruby gem that convert JSON to Parquet
+summary: Parqueteur - A Ruby gem that convert data to Parquet
 test_files: []

data/example.rb

@@ -1,39 +0,0 @@
-require 'bundler/setup'
-require 'parqueteur'
-
-class Foo < Parqueteur::Converter
-  column :id, :bigint
-  column :reference, :string
-  column :hash, :map, key: :string, value: :string
-  column :valid, :boolean
-  column :total, :integer
-  column :numbers, :array, elements: :integer
-  column :my_struct, :struct do
-    field :test, :string
-    field :mon_nombre, :integer
-  end
-end
-
-LETTERS = ('a'..'z').to_a
-
-data = 1000.times.collect do |i|
-  {
-    'id' => i + 1,
-    'reference' => "coucou:#{i}",
-    'hash' => { 'a' => LETTERS.sample },
-    'valid' => rand < 0.5,
-    'total' => rand(100..500),
-    'numbers' => [1, 2, 3],
-    'my_struct' => {
-      'test' => 'super'
-    }
-  }
-end
-
-# chunked_converter = Parqueteur::ChunkedConverter.new(data, Foo)
-# pp chunked_converter.write_files('test')
-puts Foo.convert(data, output: 'tmp/test.parquet')
-table = Arrow::Table.load('tmp/test.parquet')
-table.each_record do |record|
-  puts record.to_h
-end

data/lib/parqueteur/chunked_converter.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Parqueteur
-  class ChunkedConverter
-    attr_reader :schema
-
-    def initialize(input, converter, chunk_size = 200)
-      @input = Parqueteur::Input.from(input)
-      @converter = converter
-      @chunk_size = chunk_size
-    end
-
-    def chunks
-      Enumerator.new do |arr|
-        @input.each_slice(@chunk_size) do |chunk|
-          local_converter = @converter.new(chunk)
-          arr << local_converter.to_io
-        end
-      end
-    end
-
-    def write_files(prefix)
-      chunks.each_with_index do |chunk, idx|
-        File.write("#{prefix}.#{idx}.parquet", chunk.read)
-      end
-    end
-  end
-end

data/test.json

@@ -1 +0,0 @@
-[{"id":1,"reference":"coucou","hash":{"a":"b"},"valid":true,"hash2":{},"numbers":[1,2,3],"map_array":[]},{"id":2,"reference":"coucou","hash":{"c":"d"},"valid":false,"hash2":{},"numbers":[4,5,6],"map_array":[]},{"id":3,"reference":"coucou","hash":{"e":"f"},"valid":true,"hash2":{"x":[1,2,3]},"numbers":[7,8,9],"map_array":[{"x":"y"}]}]