parqueteur 1.0.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b60aefc8e90564af2abeeb01a36aaaabe8cf08c65efeebdbf6f034372c8100c1
-  data.tar.gz: b4f93aeac25321e50fdf8d2a8c8e0216feb4ec68d0ac356d2ed98118820c61a4
+  metadata.gz: 056b9208a8bffcd163464dbd2cf276a9b0704e96788b77555d545eb339a4e798
+  data.tar.gz: 1e20d31b1fc6f198fee42546939ce289d71d66f65ffa66562cdd7841e0f24f61
 SHA512:
-  metadata.gz: 1dbb0d1a870ff2291909014f595a6c63a35c11c80adc4e1972e20852ac87ba20447ce31dee9feb2f11dcea1a2ec5276ad4a05ca3eb406ab26a2b606ea369b809
-  data.tar.gz: ef4dec6ca81564972112468dc1729053c9c03bdc61a947a01990c1a870dc45e8136561553390fbfbb1b49d22ca1f8221a94cb8403a7ecfc6bf18f10ba07acd9b
+  metadata.gz: fe08a7b282c4ededc08acb5aa9f4b485ead828aee4fd1444e8bb1af80cc56ea8c20411aefe136809f91ad808bee52db261218e8b5e6b7538bfa53d1eb38eb4b5
+  data.tar.gz: 0fee8ec94698b7b4c9d3a089fd0094a52bd83dfda56d0652f8a5b08dfe84a88b251736e62a9da7f510e0fa3d1842e2551161178ce30b5e0f5c6ee9b903917a2c

data/.gitignore

@@ -6,3 +6,6 @@
 /pkg/
 /spec/reports/
 /tmp/
+/*.gem
+/tmp/*
+!/tmp/.keep

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parqueteur (1.0.2)
+    parqueteur (1.2.0)
       red-parquet (~> 5.0)
 
 GEM

data/examples/convert-methods.rb

@@ -0,0 +1,54 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+converter = Foo.new(data, compression: :gzip)
+pp converter.to_io
+pp converter.to_arrow_table
+converter.write('tmp/test.parquet')

data/examples/convert-to-io.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+io = Foo.convert(data)
+pp io.read

data/examples/convert-with-chunks.rb

@@ -0,0 +1,54 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+converter = Foo.new(data, compression: :gzip)
+converter.split(200).each_with_index do |chunk, idx|
+  puts "#{idx}: #{chunk.path}"
+end

data/examples/convert-with-compression.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+path = 'tmp/test.parquet'
+Foo.convert_to(data, path, compression: :gzip)

data/examples/convert-without-compression.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'parqueteur'
+require 'securerandom'
+require 'benchmark'
+
+class Foo < Parqueteur::Converter
+  column :id, :bigint
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  # column :hash2, :map, key: :string, value: :string
+  # column :hash3, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+  column :numbers, :array, elements: :integer
+  column :my_struct, :struct do
+    field :test, :string
+    field :mon_nombre, :integer
+  end
+end
+
+def random_hash
+  {
+    'a' => SecureRandom.hex(128),
+    'b' => SecureRandom.hex(128),
+    'c' => SecureRandom.hex(128),
+    'd' => SecureRandom.hex(128),
+    'e' => SecureRandom.hex(128),
+    'f' => SecureRandom.hex(128),
+    'g' => SecureRandom.hex(128),
+    'h' => SecureRandom.hex(128),
+    'i' => SecureRandom.hex(128),
+    'j' => SecureRandom.hex(128),
+    'k' => SecureRandom.hex(128),
+  }
+end
+
+data = 10000.times.collect do |i|
+  {
+    'id' => i + 1,
+    'reference' => "coucou:#{i}",
+    'hash' => random_hash,
+    # 'hash2' => random_hash,
+    # 'hash3' => random_hash,
+    'valid' => rand < 0.5,
+    'total' => rand(100..500),
+    'numbers' => [1, 2, 3]
+  }
+end
+puts "data generation OK"
+
+path = 'tmp/test.parquet'
+Foo.convert_to(data, path)

data/lib/parqueteur/column.rb

@@ -4,31 +4,14 @@ module Parqueteur
   class Column
     attr_reader :name, :type, :options
 
-    def initialize(name, type, options = {})
+    def initialize(name, type, options = {}, &block)
       @name = name.to_s
-      @type = type
+      @type = Parqueteur::TypeResolver.resolve(type, options, &block)
       @options = options
     end
 
     def arrow_type
-      @arrow_type ||= Parqueteur::TypeResolver.resolve(@type, @options)
-    end
-
-    def cast(value)
-      case @type
-      when :string then value.to_s
-      when :boolean then value == true
-      when :integer then value.to_i
-      when :long then value.to_i
-      when :timestamp
-        case value
-        when String then Time.parse(value).to_i
-        when Integer then value
-        else
-          raise ArgumentError, "Unable to cast '#{value}' to timestamp"
-        end
-      when :map then value
-      end
+      @type.arrow_type
     end
 
     def to_arrow_field

data/lib/parqueteur/column_collection.rb

@@ -4,11 +4,18 @@ module Parqueteur
   class ColumnCollection
     include Enumerable
 
+    attr_reader :column_names
+
     def initialize
       @columns = []
+      @column_names = []
       @columns_idx = {}
     end
 
+    def key?(key)
+      @columns_idx.key?(key)
+    end
+
     def each(&block)
       @columns.each(&block)
     end
@@ -17,6 +24,7 @@ module Parqueteur
       unless @columns_idx.key?(column.name)
         @columns_idx[column.name] = column
         @columns << column
+        @column_names << column.name
       end
 
       true

data/lib/parqueteur/converter.rb

@@ -2,7 +2,7 @@
 
 module Parqueteur
   class Converter
-    attr_reader :schema
+    DEFAULT_BATCH_SIZE = 10
 
     def self.inline(&block)
       Class.new(self, &block)
@@ -12,104 +12,137 @@ module Parqueteur
       @columns ||= Parqueteur::ColumnCollection.new
     end
 
-    def self.column(name, type, options = {})
-      columns.add(Parqueteur::Column.new(name, type, options))
+    def self.column(name, type, options = {}, &block)
+      columns.add(Parqueteur::Column.new(name, type, options, &block))
     end
 
     def self.transforms
       @transforms ||= []
     end
 
-    def self.transform(method_name, &block)
+    def self.transform(method_name = nil, &block)
       transforms << (method_name || block)
     end
 
-    def self.convert(input, output: nil)
-      converter = new(input)
-      if !output.nil?
-        converter.write(output)
-      else
-        converter.to_blob
+    def self.convert(input, **kwargs)
+      new(input, **kwargs).to_io
+    end
+
+    def self.convert_to(input, output_path, **kwargs)
+      converter = new(input, **kwargs)
+      converter.write(output_path)
+    end
+
+    # @param [Enumerable] An enumerable object
+    # @option [Symbol] compression - :gzip
+    def initialize(input, **kwargs)
+      @input = Parqueteur::Input.from(input)
+      @batch_size = kwargs.fetch(:batch_size, DEFAULT_BATCH_SIZE)
+      @compression = kwargs.fetch(:compression, nil)&.to_sym
+    end
+
+    def split(size)
+      Enumerator.new do |arr|
+        @input.each_slice(size) do |records|
+          local_converter = self.class.new(
+            records, batch_size: @batch_size, compression: @compression
+          )
+          file = local_converter.to_tmpfile
+          arr << file
+          file.close
+          file.unlink
+        end
       end
     end
 
-    def initialize(input, options = {})
-      @input = Parqueteur::Input.from(input, options)
-    end
-
-    def write(output)
-      case output
-      when :io
-        to_io
-      when String
-        to_arrow_table.save(output)
-      when StringIO, IO
-        buffer = Arrow::ResizableBuffer.new(0)
-        to_arrow_table.save(buffer, format: :parquet)
-        output.write(buffer.data.to_s)
-        output.rewind
-        output
-      else
-        raise ArgumentError, "unsupported output: #{output.class}, accepted: String (filename), IO, StringIO"
+    def split_by_io(size)
+      Enumerator.new do |arr|
+        @input.each_slice(size) do |records|
+          local_converter = self.class.new(records)
+          arr << local_converter.to_io
+        end
       end
     end
 
-    def to_s
-      inspect
+    def write(path)
+      arrow_schema = self.class.columns.arrow_schema
+      writer_properties = Parquet::WriterProperties.new
+      writer_properties.set_compression(@compression) unless @compression.nil?
+
+      Arrow::FileOutputStream.open(path, false) do |output|
+        Parquet::ArrowFileWriter.open(arrow_schema, output, writer_properties) do |writer|
+          @input.each_slice(@batch_size) do |records|
+            arrow_table = build_arrow_table(records)
+            writer.write_table(arrow_table, 1024)
+          end
+        end
+      end
+
+      true
+    end
+
+    def to_tmpfile
+      tempfile = Tempfile.new
+      tempfile.binmode
+      write(tempfile.path)
+      tempfile.rewind
+      tempfile
     end
 
     def to_io
-      write(StringIO.new)
+      tmpfile = to_tmpfile
+      strio = StringIO.new(tmpfile.read)
+      tmpfile.close
+      tmpfile.unlink
+      strio
+    end
+
+    def to_arrow_table
+      file = to_tmpfile
+      table = Arrow::Table.load(file.path, format: :parquet)
+      file.close
+      file.unlink
+      table
     end
 
     def to_blob
-      write(StringIO.new).read
+      to_io.read
     end
 
-    def to_arrow_table
-      transforms = self.class.transforms
+    private
 
-      chunks = {}
-      @input.each_slice(100) do |items|
-        values = self.class.columns.each_with_object({}) do |column, hash|
-          hash[column.name] = []
-        end
+    def build_arrow_table(records)
+      transforms = self.class.transforms
 
-        items.each do |item|
-          if transforms.length > 0
-            transforms.each do |transform|
-              item = \
-                if transform.is_a?(Symbol)
-                  __send__(transform, item)
-                else
-                  transform.call(item)
-                end
-            end
-          end
+      values = self.class.columns.each_with_object({}) do |column, hash|
+        hash[column.name] = []
+      end
 
-          values.each_key do |value_key|
-            if item.key?(value_key)
-              values[value_key] << item[value_key]
-            else
-              values[value_key] << nil
-            end
+      records.each do |item|
+        if transforms.length > 0
+          transforms.each do |transform|
+            item = \
+              if transform.is_a?(Symbol)
+                __send__(transform, item)
+              else
+                transform.call(item)
+              end
           end
         end
 
-        values.each_with_object(chunks) do |item, hash|
-          column = self.class.columns.find(item[0])
-          hash[item[0]] ||= []
-          hash[item[0]].push(
-            Parqueteur::ValueArrayBuilder.build(
-              item[1], column.type, column.options
-            )
-          )
+        values.each_key do |value_key|
+          if item.key?(value_key)
+            values[value_key] << item[value_key]
+          else
+            values[value_key] << nil
+          end
         end
       end
 
       Arrow::Table.new(
-        chunks.transform_values! do |value|
-          Arrow::ChunkedArray.new(value)
+        values.each_with_object({}) do |item, hash|
+          column = self.class.columns.find(item[0])
+          hash[item[0]] = column.type.build_value_array(item[1])
         end
       )
     end

data/lib/parqueteur/input.rb

@@ -4,40 +4,25 @@ module Parqueteur
   class Input
     include Enumerable
 
-    def self.from(arg, options = {})
-      new(
-        case arg
-        when String
-          if File.exist?(arg)
-            File.new(arg, 'r')
-          else
-            arg.split("\n")
-          end
-        when Array, Enumerator
-          arg
-        end,
-        options
-      )
+    def self.from(arg)
+      return arg if arg.is_a?(self)
+
+      new(arg)
     end
 
-    def initialize(source, options = {})
+    def initialize(source)
+      unless source.is_a?(Enumerable)
+        raise ArgumentError, 'Enumerable object expected'
+      end
+
       @source = source
-      @options = options
     end
 
     def each(&block)
-      case @source
-      when File
-        if @options.fetch(:json_newlines, true) == true
-          @source.each_line do |line|
-            yield(JSON.parse(line.strip))
-          end
-        else
-          JSON.parse(@source.read).each(&block)
-        end
-        @source.rewind
-      when Array, Enumerator
+      if block_given?
         @source.each(&block)
+      else
+        @source.to_enum(:each)
       end
     end
   end

data/lib/parqueteur/struct.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class Struct
+    def initialize(&block)
+      instance_exec(&block)
+    end
+
+    def fields
+      @fields ||= Parqueteur::ColumnCollection.new
+    end
+
+    def field(name, type, options = {}, &block)
+      fields.add(Parqueteur::Column.new(name, type, options, &block))
+    end
+
+    def key?(key)
+      fields.key?(key)
+    end
+
+    def to_arrow_type
+      fields.collect(&:to_arrow_field)
+    end
+  end
+end

data/lib/parqueteur/type.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class Type
+    attr_reader :options, :arrow_type
+
+    def initialize(options = {}, &block)
+      @options = options
+      @block = block
+      @arrow_type = arrow_type_builder
+    end
+
+    def build_value_array(values)
+      raise "#to_arrow_field must be implemented in #{self.class}"
+    end
+
+    def resolve(type, options = {})
+      Parqueteur::TypeResolver.resolve(type, options)
+    end
+  end
+end

data/lib/parqueteur/type_resolver.rb

@@ -2,60 +2,46 @@
 
 module Parqueteur
   class TypeResolver
-    def self.resolve(*args)
-      new.resolve(*args)
+    include Singleton
+
+    def self.registered_types
+      @registered_types ||= {
+        array: Parqueteur::Types::ArrayType,
+        bigint: Parqueteur::Types::Int64Type,
+        boolean: Parqueteur::Types::BooleanType,
+        int32: Parqueteur::Types::Int32Type,
+        int64: Parqueteur::Types::Int64Type,
+        integer: Parqueteur::Types::Int32Type,
+        map: Parqueteur::Types::MapType,
+        string: Parqueteur::Types::StringType,
+        struct: Parqueteur::Types::StructType,
+        timestamp: Parqueteur::Types::TimestampType
+      }
+    end
+
+    def self.register_type(type, klass)
+      registered_types[type] = klass
+    end
+
+    def self.resolve(*args, &block)
+      instance.resolve(*args, &block)
     end
 
-    def resolve(type, options = {})
-      case type
-      when :array
-        elements_opt = options.fetch(:elements)
-        Arrow::ListDataType.new(
-          if elements_opt.is_a?(Hash)
-            resolve(elements_opt.fetch(:type), elements_opt)
-          else
-            resolve(elements_opt)
-          end
-        )
-      when :boolean
-        Arrow::BooleanDataType.new
-      when :integer
-        if options.fetch(:unsigned, false) == true
-          Arrow::UInt32DataType.new
-        else
-          Arrow::Int32DataType.new
-        end
-      when :long
-        if options.fetch(:unsigned, false) == true
-          Arrow::UInt64DataType.new
-        else
-          Arrow::Int64DataType.new
-        end
-      when :timestamp
-        Arrow::TimestampDataType.new(
-          options.fetch(:unit, :second)
-        )
-      when :string
-        Arrow::StringDataType.new
-      when :map
-        map_value = options.fetch(:value)
-        Arrow::MapDataType.new(
-          resolve(options.fetch(:key)),
-          if map_value.is_a?(Hash)
-            resolve(map_value.fetch(:type), map_value)
-          else
-            resolve(map_value)
-          end
-        )
+    def resolve(type, options = {}, &block)
+      if type.is_a?(Symbol)
+        resolve_from_symbol(type, options, &block)
       else
-        raise Error, "unknown type: #{type}"
+        type.new(options, &block)
       end
     end
-  end
-end
 
-private
+    private
 
-def build_arrow_type(type, options = {})
+    def resolve_from_symbol(type, options, &block)
+      type_klass = self.class.registered_types.fetch(type.to_sym, nil)
+      raise Parqueteur::TypeNotFound, type if type_klass.nil?
 
+      type_klass.new(options, &block)
+    end
+  end
 end

data/lib/parqueteur/types/array_type.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class ArrayType < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::ListArrayBuilder.build(arrow_type, values)
+      end
+
+      def arrow_type_builder
+        Arrow::ListDataType.new(
+          if options[:elements].is_a?(Hash)
+            resolve(options[:elements].fetch(:type), options[:elements]).arrow_type
+          else
+            resolve(options[:elements]).arrow_type
+          end
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/boolean_type.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class BooleanType < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::BooleanArray.new(values)
+      end
+
+      def arrow_type_builder
+        Arrow::BooleanDataType.new
+      end
+    end
+  end
+end

data/lib/parqueteur/types/int32_type.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Int32Type < Parqueteur::Type
+      def build_value_array(values)
+        if options.fetch(:unsigned, false) == true
+          Arrow::UInt32Array.new(values)
+        else
+          Arrow::Int32Array.new(values)
+        end
+      end
+
+      def arrow_type_builder
+        if options.fetch(:unsigned, false) == true
+          Arrow::UInt32DataType.new
+        else
+          Arrow::Int32DataType.new
+        end
+      end
+    end
+  end
+end
+
+# when :integer

data/lib/parqueteur/types/int64_type.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Int64Type < Parqueteur::Type
+      def build_value_array(values)
+        if options.fetch(:unsigned, false) == true
+          Arrow::UInt64Array.new(values)
+        else
+          Arrow::Int64Array.new(values)
+        end
+      end
+
+      def arrow_type_builder
+        if options.fetch(:unsigned, false) == true
+          Arrow::UInt64DataType.new
+        else
+          Arrow::Int64DataType.new
+        end
+      end
+    end
+  end
+end
+
+# when :integer

data/lib/parqueteur/types/map_type.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class MapType < Parqueteur::Type
+      def build_value_array(values)
+        builder = Arrow::MapArrayBuilder.new(arrow_type)
+        values.each do |entry|
+          builder.append_value
+          next if entry.nil?
+
+          entry.each do |k, v|
+            builder.key_builder.append(k)
+            builder.item_builder.append(v)
+          end
+        end
+
+        builder.finish
+      end
+
+      def arrow_type_builder
+        map_value = options.fetch(:value)
+
+        Arrow::MapDataType.new(
+          resolve(options.fetch(:key)).arrow_type,
+          if map_value.is_a?(Hash)
+            resolve(map_value.fetch(:type), map_value).arrow_type
+          else
+            resolve(map_value).arrow_type
+          end
+        )
+      end
+    end
+  end
+end
+

data/lib/parqueteur/types/string_type.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# when :timestamp
+#   Arrow::TimestampDataType.new(
+#     options.fetch(:unit, :second)
+#   )
+
+module Parqueteur
+  module Types
+    class StringType < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::StringArray.new(values)
+      end
+
+      def arrow_type_builder
+        Arrow::StringDataType.new
+      end
+    end
+  end
+end

data/lib/parqueteur/types/struct_type.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# when :timestamp
+#   Arrow::TimestampDataType.new(
+#     options.fetch(:unit, :second)
+#   )
+
+module Parqueteur
+  module Types
+    class StructType < Parqueteur::Type
+      def build_value_array(values)
+        values.each do |value|
+          next if value.nil?
+
+          value.each_key do |key|
+            next if struct_object.key?(key)
+
+            raise Parqueteur::Error, "Struct field '#{key}' not found"
+          end
+        end
+        Arrow::StructArrayBuilder.build(arrow_type, values)
+      end
+
+      def arrow_type_builder
+        Arrow::StructDataType.new(struct_object.to_arrow_type)
+      end
+
+      private
+
+      def struct_object
+        @struct_object ||= Parqueteur::Struct.new(&@block)
+      end
+    end
+  end
+end

data/lib/parqueteur/types/timestamp_type.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# when :timestamp
+#   Arrow::TimestampDataType.new(
+#     options.fetch(:unit, :second)
+#   )
+
+module Parqueteur
+  module Types
+    class TimestampType < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::TimestampArray.new(values)
+      end
+
+      def arrow_type_builder
+        Arrow::TimestampDataType.new(
+          options.fetch(:unit, :second)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parqueteur
-  VERSION = '1.0.2'
+  VERSION = '1.2.0'
 end

data/lib/parqueteur.rb

@@ -1,17 +1,29 @@
 # frozen_string_literal: true
 
-require_relative "parqueteur/version"
-require 'parqueteur/type_resolver'
+require 'json'
+require 'singleton'
+require 'tempfile'
+
+require_relative 'parqueteur/version'
 require 'parqueteur/column'
 require 'parqueteur/column_collection'
 require 'parqueteur/converter'
-require 'parqueteur/chunked_converter'
 require 'parqueteur/input'
-require 'parqueteur/value_array_builder'
-require 'json'
+require 'parqueteur/struct'
+require 'parqueteur/type'
+require 'parqueteur/type_resolver'
+require 'parqueteur/types/array_type'
+require 'parqueteur/types/boolean_type'
+require 'parqueteur/types/int32_type'
+require 'parqueteur/types/int64_type'
+require 'parqueteur/types/map_type'
+require 'parqueteur/types/string_type'
+require 'parqueteur/types/struct_type'
+require 'parqueteur/types/timestamp_type'
 require 'parquet'
 
 module Parqueteur
   class Error < StandardError; end
+  class TypeNotFound < Error; end
   # Your code goes here...
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parqueteur
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Julien D.
@@ -38,15 +38,27 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- example.rb
+- examples/convert-methods.rb
+- examples/convert-to-io.rb
+- examples/convert-with-chunks.rb
+- examples/convert-with-compression.rb
+- examples/convert-without-compression.rb
 - lib/parqueteur.rb
-- lib/parqueteur/chunked_converter.rb
 - lib/parqueteur/column.rb
 - lib/parqueteur/column_collection.rb
 - lib/parqueteur/converter.rb
 - lib/parqueteur/input.rb
+- lib/parqueteur/struct.rb
+- lib/parqueteur/type.rb
 - lib/parqueteur/type_resolver.rb
-- lib/parqueteur/value_array_builder.rb
+- lib/parqueteur/types/array_type.rb
+- lib/parqueteur/types/boolean_type.rb
+- lib/parqueteur/types/int32_type.rb
+- lib/parqueteur/types/int64_type.rb
+- lib/parqueteur/types/map_type.rb
+- lib/parqueteur/types/string_type.rb
+- lib/parqueteur/types/struct_type.rb
+- lib/parqueteur/types/timestamp_type.rb
 - lib/parqueteur/version.rb
 - parqueteur.gemspec
 - test.json

data/example.rb

@@ -1,20 +0,0 @@
-require 'bundler/setup'
-require 'parqueteur'
-
-class Foo < Parqueteur::Converter
-  column :id, :long
-  column :reference, :string
-  column :hash, :map, key: :string, value: :string
-  column :valid, :boolean
-  column :total, :integer
-end
-
-LETTERS = ('a'..'z').to_a
-
-data = 1000.times.collect do |i|
-  { 'id' => i + 1, 'reference' => "coucou:#{i}", 'hash' => { 'a' => LETTERS.sample }, 'valid' => rand < 0.5, 'total' => rand(100..500) }
-end
-
-chunked_converter = Parqueteur::ChunkedConverter.new(data, Foo)
-pp chunked_converter.write_files('test')
-# puts Foo.convert(data, output: 'test.parquet')

data/lib/parqueteur/chunked_converter.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Parqueteur
-  class ChunkedConverter
-    attr_reader :schema
-
-    def initialize(input, converter, chunk_size = 200)
-      @input = Parqueteur::Input.from(input)
-      @converter = converter
-      @chunk_size = chunk_size
-    end
-
-    def chunks
-      Enumerator.new do |arr|
-        @input.each_slice(@chunk_size) do |chunk|
-          local_converter = @converter.new(chunk)
-          arr << local_converter.to_io
-        end
-      end
-    end
-
-    def write_files(prefix)
-      chunks.each_with_index do |chunk, idx|
-        File.write("#{prefix}.#{idx}.parquet", chunk.read)
-      end
-    end
-  end
-end

data/lib/parqueteur/value_array_builder.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module Parqueteur
-  class ValueArrayBuilder
-    attr_reader :type, :options, :arrow_type
-
-    def self.build(input, type, options)
-      new(type, options).build(input)
-    end
-
-    def initialize(type, options)
-      @type = type
-      @options = options
-      @arrow_type = Parqueteur::TypeResolver.resolve(type, options)
-    end
-
-    def build(input)
-      return if input.nil?
-
-      case type
-      when :array
-        Arrow::ListArrayBuilder.build(arrow_type, input)
-      when :map
-        builder = Arrow::MapArrayBuilder.new(arrow_type)
-        input.each do |entry|
-          builder.append_value
-          next if entry.nil?
-
-          entry.each do |k, v|
-            builder.key_builder.append(k)
-            builder.item_builder.append(v)
-          end
-        end
-
-        builder.finish
-      when :boolean
-        Arrow::BooleanArray.new(input)
-      when :integer
-        if options.fetch(:unsigned, false) == true
-          Arrow::UInt32Array.new(input)
-        else
-          Arrow::Int32Array.new(input)
-        end
-      when :long
-        if options.fetch(:unsigned, false) == true
-          Arrow::UInt64Array.new(input)
-        else
-          Arrow::Int64Array.new(input)
-        end
-      when :string
-        Arrow::StringArray.new(input)
-      when :timestamp
-        Arrow::TimestampArray.new(input)
-      else
-        raise Error, "unknown type: #{type}"
-      end
-    end
-  end
-end