parqueteur 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1805a7356005e238c3d8f4318a54ea498ff58a0fcd93819c51cc445c4f8b6ee0
-  data.tar.gz: 1360c3d481eac282e684f588e49cbcd142c7cc0f6c345d8a98e48e402a9edf22
+  metadata.gz: 056b9208a8bffcd163464dbd2cf276a9b0704e96788b77555d545eb339a4e798
+  data.tar.gz: 1e20d31b1fc6f198fee42546939ce289d71d66f65ffa66562cdd7841e0f24f61
 SHA512:
-  metadata.gz: 28c157c39f25744a10223e8d11d165b7e4e25eb70e45cf2e22513795b4b1b4b1b9f44bc04d5ef8e10b74ebd82d1c87bc318a02b32fc25dbd6c32c0763e79801c
-  data.tar.gz: 6f0433e8090c49c48278bcd4d5f36f96c394bd8d8386fbf2bc22d224d42ef58036beaa0307c32420cbbb3b33eec463964b28e616f862385de6afa1f8c98c9310
+  metadata.gz: fe08a7b282c4ededc08acb5aa9f4b485ead828aee4fd1444e8bb1af80cc56ea8c20411aefe136809f91ad808bee52db261218e8b5e6b7538bfa53d1eb38eb4b5
+  data.tar.gz: 0fee8ec94698b7b4c9d3a089fd0094a52bd83dfda56d0652f8a5b08dfe84a88b251736e62a9da7f510e0fa3d1842e2551161178ce30b5e0f5c6ee9b903917a2c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parqueteur (1.1.1)
+    parqueteur (1.2.0)
       red-parquet (~> 5.0)
 
 GEM

data/examples/convert-methods.rb

@@ -0,0 +1,54 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+converter = Foo.new(data, compression: :gzip)
+pp converter.to_io
+pp converter.to_arrow_table
+converter.write('tmp/test.parquet')

data/examples/convert-to-io.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+io = Foo.convert(data)
+pp io.read

data/examples/convert-with-chunks.rb

@@ -0,0 +1,54 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+converter = Foo.new(data, compression: :gzip)
+converter.split(200).each_with_index do |chunk, idx|
+  puts "#{idx}: #{chunk.path}"
+end

data/examples/convert-with-compression.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+path = 'tmp/test.parquet'
+Foo.convert_to(data, path, compression: :gzip)

data/examples/convert-without-compression.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+path = 'tmp/test.parquet'
+Foo.convert_to(data, path)

data/lib/parqueteur/converter.rb

@@ -2,9 +2,7 @@
 
 module Parqueteur
   class Converter
-    attr_reader :schema
-
-    DEFAULT_BATCH_SIZE = 25
+    DEFAULT_BATCH_SIZE = 10
 
     def self.inline(&block)
       Class.new(self, &block)
@@ -26,106 +24,127 @@ module Parqueteur
       transforms << (method_name || block)
     end
 
-    def self.convert(input, output: nil, batch_size: DEFAULT_BATCH_SIZE)
-      converter = new(input, bactch_size: batch_size)
-      if !output.nil?
-        converter.write(output)
-      else
-        converter.to_blob
+    def self.convert(input, **kwargs)
+      new(input, **kwargs).to_io
+    end
+
+    def self.convert_to(input, output_path, **kwargs)
+      converter = new(input, **kwargs)
+      converter.write(output_path)
+    end
+
+    # @param [Enumerable] An enumerable object
+    # @option [Symbol] compression - :gzip
+    def initialize(input, **kwargs)
+      @input = Parqueteur::Input.from(input)
+      @batch_size = kwargs.fetch(:batch_size, DEFAULT_BATCH_SIZE)
+      @compression = kwargs.fetch(:compression, nil)&.to_sym
+    end
+
+    def split(size)
+      Enumerator.new do |arr|
+        @input.each_slice(size) do |records|
+          local_converter = self.class.new(
+            records, batch_size: @batch_size, compression: @compression
+          )
+          file = local_converter.to_tmpfile
+          arr << file
+          file.close
+          file.unlink
+        end
       end
     end
 
-    def initialize(input, options = {})
-      @input = Parqueteur::Input.from(input, options)
-      @batch_size = options.fetch(:batch_size, DEFAULT_BATCH_SIZE)
-    end
-
-    def write(output)
-      case output
-      when :io
-        to_io
-      when String
-        to_arrow_table.save(output)
-      when StringIO, IO
-        buffer = Arrow::ResizableBuffer.new(0)
-        to_arrow_table.save(buffer, format: :parquet)
-        output.write(buffer.data.to_s)
-        output.rewind
-        output
-      else
-        raise ArgumentError, "unsupported output: #{output.class}, accepted: String (filename), IO, StringIO"
+    def split_by_io(size)
+      Enumerator.new do |arr|
+        @input.each_slice(size) do |records|
+          local_converter = self.class.new(records)
+          arr << local_converter.to_io
+        end
+      end
+    end
+
+    def write(path)
+      arrow_schema = self.class.columns.arrow_schema
+      writer_properties = Parquet::WriterProperties.new
+      writer_properties.set_compression(@compression) unless @compression.nil?
+
+      Arrow::FileOutputStream.open(path, false) do |output|
+        Parquet::ArrowFileWriter.open(arrow_schema, output, writer_properties) do |writer|
+          @input.each_slice(@batch_size) do |records|
+            arrow_table = build_arrow_table(records)
+            writer.write_table(arrow_table, 1024)
+          end
+        end
       end
+
+      true
     end
 
-    def to_s
-      inspect
+    def to_tmpfile
+      tempfile = Tempfile.new
+      tempfile.binmode
+      write(tempfile.path)
+      tempfile.rewind
+      tempfile
     end
 
     def to_io
-      write(StringIO.new)
+      tmpfile = to_tmpfile
+      strio = StringIO.new(tmpfile.read)
+      tmpfile.close
+      tmpfile.unlink
+      strio
+    end
+
+    def to_arrow_table
+      file = to_tmpfile
+      table = Arrow::Table.load(file.path, format: :parquet)
+      file.close
+      file.unlink
+      table
     end
 
     def to_blob
-      write(StringIO.new).read
+      to_io.read
     end
 
-    def to_arrow_table
+    private
+
+    def build_arrow_table(records)
       transforms = self.class.transforms
 
-      chunks = self.class.columns.each_with_object({}) do |column, hash|
+      values = self.class.columns.each_with_object({}) do |column, hash|
         hash[column.name] = []
       end
 
-      items_count = 0
-      @input.each_slice(@batch_size) do |items|
-        values = self.class.columns.each_with_object({}) do |column, hash|
-          hash[column.name] = []
-        end
-
-        items.each do |item|
-          if transforms.length > 0
-            transforms.each do |transform|
-              item = \
-                if transform.is_a?(Symbol)
-                  __send__(transform, item)
-                else
-                  transform.call(item)
-                end
-            end
+      records.each do |item|
+        if transforms.length > 0
+          transforms.each do |transform|
+            item = \
+              if transform.is_a?(Symbol)
+                __send__(transform, item)
+              else
+                transform.call(item)
+              end
           end
+        end
 
-          values.each_key do |value_key|
-            if item.key?(value_key)
-              values[value_key] << item[value_key]
-            else
-              values[value_key] << nil
-            end
+        values.each_key do |value_key|
+          if item.key?(value_key)
+            values[value_key] << item[value_key]
+          else
+            values[value_key] << nil
           end
         end
+      end
 
-        values.each_with_object(chunks) do |item, hash|
+      Arrow::Table.new(
+        values.each_with_object({}) do |item, hash|
           column = self.class.columns.find(item[0])
-          hash[item[0]].push(
-            column.type.build_value_array(item[1])
-          )
+          hash[item[0]] = column.type.build_value_array(item[1])
         end
-
-        items_count += items.length
-      end
-
-      if items_count > 0
-        Arrow::Table.new(
-          chunks.transform_values! do |value|
-            Arrow::ChunkedArray.new(value)
-          end
-        )
-      else
-        Arrow::Table.new(
-          self.class.columns.each_with_object({}) do |column, hash|
-            hash[column.name] = column.type.build_value_array([])
-          end
-        )
-      end
+      )
     end
   end
 end

data/lib/parqueteur/input.rb

@@ -4,40 +4,25 @@ module Parqueteur
   class Input
     include Enumerable
 
-    def self.from(arg, options = {})
-      new(
-        case arg
-        when String
-          if File.exist?(arg)
-            File.new(arg, 'r')
-          else
-            arg.split("\n")
-          end
-        when Enumerable
-          arg
-        end,
-        options
-      )
+    def self.from(arg)
+      return arg if arg.is_a?(self)
+
+      new(arg)
     end
 
-    def initialize(source, options = {})
+    def initialize(source)
+      unless source.is_a?(Enumerable)
+        raise ArgumentError, 'Enumerable object expected'
+      end
+
       @source = source
-      @options = options
     end
 
     def each(&block)
-      case @source
-      when File
-        if @options.fetch(:json_newlines, true) == true
-          @source.each_line do |line|
-            yield(JSON.parse(line.strip))
-          end
-        else
-          JSON.parse(@source.read).each(&block)
-        end
-        @source.rewind
-      when Enumerable
+      if block_given?
         @source.each(&block)
+      else
+        @source.to_enum(:each)
       end
     end
   end

data/lib/parqueteur/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parqueteur
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 end

data/lib/parqueteur.rb

@@ -2,9 +2,9 @@
 
 require 'json'
 require 'singleton'
+require 'tempfile'
 
-require_relative "parqueteur/version"
-require 'parqueteur/chunked_converter'
+require_relative 'parqueteur/version'
 require 'parqueteur/column'
 require 'parqueteur/column_collection'
 require 'parqueteur/converter'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parqueteur
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Julien D.
@@ -38,9 +38,12 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- example.rb
+- examples/convert-methods.rb
+- examples/convert-to-io.rb
+- examples/convert-with-chunks.rb
+- examples/convert-with-compression.rb
+- examples/convert-without-compression.rb
 - lib/parqueteur.rb
-- lib/parqueteur/chunked_converter.rb
 - lib/parqueteur/column.rb
 - lib/parqueteur/column_collection.rb
 - lib/parqueteur/converter.rb

data/example.rb

@@ -1,39 +0,0 @@
-require 'bundler/setup'
-require 'parqueteur'
-
-class Foo < Parqueteur::Converter
-  column :id, :bigint
-  column :reference, :string
-  column :hash, :map, key: :string, value: :string
-  column :valid, :boolean
-  column :total, :integer
-  column :numbers, :array, elements: :integer
-  column :my_struct, :struct do
-    field :test, :string
-    field :mon_nombre, :integer
-  end
-end
-
-LETTERS = ('a'..'z').to_a
-
-data = 1000.times.collect do |i|
-  {
-    'id' => i + 1,
-    'reference' => "coucou:#{i}",
-    'hash' => { 'a' => LETTERS.sample },
-    'valid' => rand < 0.5,
-    'total' => rand(100..500),
-    'numbers' => [1, 2, 3],
-    'my_struct' => {
-      'test' => 'super'
-    }
-  }
-end
-
-# chunked_converter = Parqueteur::ChunkedConverter.new(data, Foo)
-# pp chunked_converter.write_files('test')
-puts Foo.convert(data, output: 'tmp/test.parquet')
-table = Arrow::Table.load('tmp/test.parquet')
-table.each_record do |record|
-  puts record.to_h
-end

data/lib/parqueteur/chunked_converter.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Parqueteur
-  class ChunkedConverter
-    attr_reader :schema
-
-    def initialize(input, converter, chunk_size = 200)
-      @input = Parqueteur::Input.from(input)
-      @converter = converter
-      @chunk_size = chunk_size
-    end
-
-    def chunks
-      Enumerator.new do |arr|
-        @input.each_slice(@chunk_size) do |chunk|
-          local_converter = @converter.new(chunk)
-          arr << local_converter.to_io
-        end
-      end
-    end
-
-    def write_files(prefix)
-      chunks.each_with_index do |chunk, idx|
-        File.write("#{prefix}.#{idx}.parquet", chunk.read)
-      end
-    end
-  end
-end