parqueteur 1.0.3 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51c58dad3007e9c6a08b4673b378975c7839158240c27fb0a883d0e59c05d5d5
-  data.tar.gz: fa63fe9cd3cfa71d587400f3672ca2697c87a77fc574d8e2463d89f0090cd686
+  metadata.gz: 715ea84521855ea8978e4f944fd6ae07f192bf97fdd40893b4b9bd292a3fe0b5
+  data.tar.gz: d05798d68479c37a8d7028cd65ff35438399de4459d84d4797399940c63513f6
 SHA512:
-  metadata.gz: 58c98870d35b4af03bf52fea1572d26b4073d16696637efe8992eba9ddfbe8931e3b015976cd997ee85fc1a0388b031ecafe1b165b911a36d6748a90ce8af88d
-  data.tar.gz: 288418ec8a410f94fe38e76d94bb2da4173a4760ed8a9825132a41f3d808370ff60c8975ea8e6b8b72c1198dd36e5b4ceb1682570edf4ae9fa60be93ddc3cba8
+  metadata.gz: 1a4d74f311c64f79c6e339ba05e631feb94552268936f58c86e0e9bcf70bb46fec8a94d452501cd395203ea33ab9440e015e4503f255cfcecc7703e6fc8d0a1b
+  data.tar.gz: 344ea6420b6c08bbe61f4f534a92c26f563193d3f6dc995dd7eb09d9df5e9f3342971bbd53570d8db0f6b910e457e62e95811c04ba2245578de3b8bb245f7dc1

data/.gitignore

@@ -7,3 +7,5 @@
 /spec/reports/
 /tmp/
 /*.gem
+/tmp/*
+!/tmp/.keep

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parqueteur (1.0.3)
+    parqueteur (1.3.0)
       red-parquet (~> 5.0)
 
 GEM

data/README.md

@@ -1,15 +1,29 @@
 # Parqueteur
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/parqueteur`. To experiment with that code, run `bin/console` for an interactive prompt.
+Parqueteur enables you to generate Apache Parquet files from raw data.
 
-TODO: Delete this and the text above, and describe your gem
+## Dependencies
 
+Since I only tested Parqueteur on Ubuntu, I don't have any install scripts for other operating systems.
+### Debian/Ubuntu packages
+- `libgirepository1.0-dev`
+- `libarrow-dev`
+- `libarrow-glib-dev`
+- `libparquet-dev`
+- `libparquet-glib-dev`
+
+You can check `scripts/apache-arrow-ubuntu-install.sh` script for a quick way to install all of them.
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'parqueteur'
+gem 'parqueteur', '~> 1.0'
+```
+
+> (optional) If you don't want to require Parqueteur globally you can add `require: false` to the Gemfile instruction:
+```ruby
+gem 'parqueteur', '~> 1.0', require: false
 ```
 
 And then execute:
@@ -22,14 +36,35 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+Parqueteur provides an elegant way to generate Apache Parquet files from a defined schema.
+```ruby
+require 'parqueteur'
 
-## Development
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+end
 
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+data = [
+  { 'id' => 1, 'reference' => 'hello world 1' },
+  { 'id' => 2, 'reference' => 'hello world 2' },
+  { 'id' => 3, 'reference' => 'hello world 3' }
+]
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('hello_world.parquet')
+
+# in-memory result (StringIO)
+converter.to_io
+
+# write to temporary file (Tempfile)
+# don't forget to `close` / `unlink` it after usage
+converter.to_tmpfile
+```
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/parqueteur.
+Bug reports and pull requests are welcome on GitHub at https://github.com/pocketsizesun/parqueteur-ruby.

data/examples/convert-and-compression.rb

@@ -0,0 +1,56 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('tmp/example.gzip-compressed.parquet')
+converter.write('tmp/example.no-gzip.parquet', compression: false)

data/examples/convert-methods.rb

@@ -0,0 +1,54 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+converter = Foo.new(data, compression: :gzip)
+pp converter.to_io
+pp converter.to_arrow_table
+converter.write('tmp/test.parquet')

data/examples/convert-to-io.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+io = Foo.convert(data)
+pp io.read

data/examples/convert-with-chunks.rb

@@ -0,0 +1,54 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+converter = Foo.new(data, compression: :gzip)
+converter.split(200).each_with_index do |chunk, idx|
+  puts "#{idx}: #{chunk.path}"
+end

data/examples/convert-without-compression.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+path = 'tmp/test.parquet'
+Foo.convert_to(data, path)

data/examples/hello-world.rb

@@ -0,0 +1,56 @@
+require 'bundler/setup'
+require 'parqueteur'
+
+class FooParquetConverter < Parqueteur::Converter
+  column :id, :bigint
+  column :my_string_array, :array, elements: :string
+  column :my_date, :date
+  column :my_decimal, :decimal, precision: 12, scale: 4
+  column :my_int, :integer
+  column :my_map, :map, key: :string, value: :string
+  column :my_string, :string
+  column :my_struct, :struct do
+    field :my_struct_str, :string
+    field :my_struct_int, :integer
+  end
+  column :my_time, :time
+  column :my_timestamp, :timestamp
+end
+
+data = 100.times.collect do |i|
+  {
+    'id' => i,
+    'my_string_array' => %w[a b c],
+    'my_date' => Date.today,
+    'my_decimal' => BigDecimal('789000.5678'),
+    'my_int' => rand(1..10),
+    'my_map' => { 'a' => 'b' },
+    'my_string' => 'Hello World',
+    'my_struct' => {
+      'my_struct_str' => 'Hello World',
+      'my_struct_int' => 1
+    },
+    'my_time' => 3600,
+    'my_timestamp' => Time.now
+  }
+end
+
+# initialize Converter with Parquet GZIP compression mode
+converter = FooParquetConverter.new(data, compression: :gzip)
+
+# write result to file
+converter.write('tmp/hello_world.compressed.parquet')
+converter.write('tmp/hello_world.parquet', compression: false)
+
+# in-memory result (StringIO)
+converter.to_io
+
+# write to temporary file (Tempfile)
+# don't forget to `close` / `unlink` it after usage
+converter.to_tmpfile
+
+# Arrow Table
+table = converter.to_arrow_table
+table.each_record do |record|
+  pp record.to_h
+end

data/lib/parqueteur/column.rb

@@ -4,31 +4,14 @@ module Parqueteur
   class Column
     attr_reader :name, :type, :options
 
-    def initialize(name, type, options = {})
+    def initialize(name, type, options = {}, &block)
       @name = name.to_s
-      @type = type
+      @type = Parqueteur::TypeResolver.resolve(type, options, &block)
       @options = options
     end
 
     def arrow_type
-      @arrow_type ||= Parqueteur::TypeResolver.resolve(@type, @options)
-    end
-
-    def cast(value)
-      case @type
-      when :string then value.to_s
-      when :boolean then value == true
-      when :integer then value.to_i
-      when :long then value.to_i
-      when :timestamp
-        case value
-        when String then Time.parse(value).to_i
-        when Integer then value
-        else
-          raise ArgumentError, "Unable to cast '#{value}' to timestamp"
-        end
-      when :map then value
-      end
+      @type.arrow_type
     end
 
     def to_arrow_field

data/lib/parqueteur/column_collection.rb

@@ -4,11 +4,18 @@ module Parqueteur
   class ColumnCollection
     include Enumerable
 
+    attr_reader :column_names
+
     def initialize
       @columns = []
+      @column_names = []
       @columns_idx = {}
     end
 
+    def key?(key)
+      @columns_idx.key?(key)
+    end
+
     def each(&block)
       @columns.each(&block)
     end
@@ -17,6 +24,7 @@ module Parqueteur
       unless @columns_idx.key?(column.name)
         @columns_idx[column.name] = column
         @columns << column
+        @column_names << column.name
       end
 
       true