parqueteur 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b60aefc8e90564af2abeeb01a36aaaabe8cf08c65efeebdbf6f034372c8100c1
+  data.tar.gz: b4f93aeac25321e50fdf8d2a8c8e0216feb4ec68d0ac356d2ed98118820c61a4
+SHA512:
+  metadata.gz: 1dbb0d1a870ff2291909014f595a6c63a35c11c80adc4e1972e20852ac87ba20447ce31dee9feb2f11dcea1a2ec5276ad4a05ca3eb406ab26a2b606ea369b809
+  data.tar.gz: ef4dec6ca81564972112468dc1729053c9c03bdc61a947a01990c1a870dc45e8136561553390fbfbb1b49d22ca1f8221a94cb8403a7ecfc6bf18f10ba07acd9b

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in parqueteur.gemspec
+gemspec
+
+gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -0,0 +1,39 @@
+PATH
+  remote: .
+  specs:
+    parqueteur (1.0.2)
+      red-parquet (~> 5.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    bigdecimal (3.0.0)
+    extpp (0.0.9)
+    gio2 (3.4.6)
+      gobject-introspection (= 3.4.6)
+    glib2 (3.4.6)
+      native-package-installer (>= 1.0.3)
+      pkg-config (>= 1.3.5)
+    gobject-introspection (3.4.6)
+      glib2 (= 3.4.6)
+    native-package-installer (1.1.1)
+    pkg-config (1.4.6)
+    rake (13.0.6)
+    red-arrow (5.0.0)
+      bigdecimal (>= 2.0.3)
+      extpp (>= 0.0.7)
+      gio2 (>= 3.4.5)
+      native-package-installer
+      pkg-config
+    red-parquet (5.0.0)
+      red-arrow (= 5.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  parqueteur!
+  rake (~> 13.0)
+
+BUNDLED WITH
+   2.2.3

data/README.md

@@ -0,0 +1,35 @@
+# Parqueteur
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/parqueteur`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'parqueteur'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install parqueteur
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/parqueteur.

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+task default: %i[]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "parqueteur"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/example.rb

@@ -0,0 +1,20 @@
+require 'bundler/setup'
+require 'parqueteur'
+
+class Foo < Parqueteur::Converter
+  column :id, :long
+  column :reference, :string
+  column :hash, :map, key: :string, value: :string
+  column :valid, :boolean
+  column :total, :integer
+end
+
+LETTERS = ('a'..'z').to_a
+
+data = 1000.times.collect do |i|
+  { 'id' => i + 1, 'reference' => "coucou:#{i}", 'hash' => { 'a' => LETTERS.sample }, 'valid' => rand < 0.5, 'total' => rand(100..500) }
+end
+
+chunked_converter = Parqueteur::ChunkedConverter.new(data, Foo)
+pp chunked_converter.write_files('test')
+# puts Foo.convert(data, output: 'test.parquet')

data/lib/parqueteur/chunked_converter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class ChunkedConverter
+    attr_reader :schema
+
+    def initialize(input, converter, chunk_size = 200)
+      @input = Parqueteur::Input.from(input)
+      @converter = converter
+      @chunk_size = chunk_size
+    end
+
+    def chunks
+      Enumerator.new do |arr|
+        @input.each_slice(@chunk_size) do |chunk|
+          local_converter = @converter.new(chunk)
+          arr << local_converter.to_io
+        end
+      end
+    end
+
+    def write_files(prefix)
+      chunks.each_with_index do |chunk, idx|
+        File.write("#{prefix}.#{idx}.parquet", chunk.read)
+      end
+    end
+  end
+end

data/lib/parqueteur/column.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class Column
+    attr_reader :name, :type, :options
+
+    def initialize(name, type, options = {})
+      @name = name.to_s
+      @type = type
+      @options = options
+    end
+
+    def arrow_type
+      @arrow_type ||= Parqueteur::TypeResolver.resolve(@type, @options)
+    end
+
+    def cast(value)
+      case @type
+      when :string then value.to_s
+      when :boolean then value == true
+      when :integer then value.to_i
+      when :long then value.to_i
+      when :timestamp
+        case value
+        when String then Time.parse(value).to_i
+        when Integer then value
+        else
+          raise ArgumentError, "Unable to cast '#{value}' to timestamp"
+        end
+      when :map then value
+      end
+    end
+
+    def to_arrow_field
+      Arrow::Field.new(name, arrow_type)
+    end
+  end
+end

data/lib/parqueteur/column_collection.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class ColumnCollection
+    include Enumerable
+
+    def initialize
+      @columns = []
+      @columns_idx = {}
+    end
+
+    def each(&block)
+      @columns.each(&block)
+    end
+
+    def add(column)
+      unless @columns_idx.key?(column.name)
+        @columns_idx[column.name] = column
+        @columns << column
+      end
+
+      true
+    end
+
+    def find(name)
+      @columns_idx.fetch(name, nil)
+    end
+
+    def arrow_schema
+      @arrow_schema ||= Arrow::Schema.new(@columns.collect(&:to_arrow_field))
+    end
+  end
+end

data/lib/parqueteur/converter.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class Converter
+    attr_reader :schema
+
+    def self.inline(&block)
+      Class.new(self, &block)
+    end
+
+    def self.columns
+      @columns ||= Parqueteur::ColumnCollection.new
+    end
+
+    def self.column(name, type, options = {})
+      columns.add(Parqueteur::Column.new(name, type, options))
+    end
+
+    def self.transforms
+      @transforms ||= []
+    end
+
+    def self.transform(method_name, &block)
+      transforms << (method_name || block)
+    end
+
+    def self.convert(input, output: nil)
+      converter = new(input)
+      if !output.nil?
+        converter.write(output)
+      else
+        converter.to_blob
+      end
+    end
+
+    def initialize(input, options = {})
+      @input = Parqueteur::Input.from(input, options)
+    end
+
+    def write(output)
+      case output
+      when :io
+        to_io
+      when String
+        to_arrow_table.save(output)
+      when StringIO, IO
+        buffer = Arrow::ResizableBuffer.new(0)
+        to_arrow_table.save(buffer, format: :parquet)
+        output.write(buffer.data.to_s)
+        output.rewind
+        output
+      else
+        raise ArgumentError, "unsupported output: #{output.class}, accepted: String (filename), IO, StringIO"
+      end
+    end
+
+    def to_s
+      inspect
+    end
+
+    def to_io
+      write(StringIO.new)
+    end
+
+    def to_blob
+      write(StringIO.new).read
+    end
+
+    def to_arrow_table
+      transforms = self.class.transforms
+
+      chunks = {}
+      @input.each_slice(100) do |items|
+        values = self.class.columns.each_with_object({}) do |column, hash|
+          hash[column.name] = []
+        end
+
+        items.each do |item|
+          if transforms.length > 0
+            transforms.each do |transform|
+              item = \
+                if transform.is_a?(Symbol)
+                  __send__(transform, item)
+                else
+                  transform.call(item)
+                end
+            end
+          end
+
+          values.each_key do |value_key|
+            if item.key?(value_key)
+              values[value_key] << item[value_key]
+            else
+              values[value_key] << nil
+            end
+          end
+        end
+
+        values.each_with_object(chunks) do |item, hash|
+          column = self.class.columns.find(item[0])
+          hash[item[0]] ||= []
+          hash[item[0]].push(
+            Parqueteur::ValueArrayBuilder.build(
+              item[1], column.type, column.options
+            )
+          )
+        end
+      end
+
+      Arrow::Table.new(
+        chunks.transform_values! do |value|
+          Arrow::ChunkedArray.new(value)
+        end
+      )
+    end
+  end
+end

data/lib/parqueteur/input.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class Input
+    include Enumerable
+
+    def self.from(arg, options = {})
+      new(
+        case arg
+        when String
+          if File.exist?(arg)
+            File.new(arg, 'r')
+          else
+            arg.split("\n")
+          end
+        when Array, Enumerator
+          arg
+        end,
+        options
+      )
+    end
+
+    def initialize(source, options = {})
+      @source = source
+      @options = options
+    end
+
+    def each(&block)
+      case @source
+      when File
+        if @options.fetch(:json_newlines, true) == true
+          @source.each_line do |line|
+            yield(JSON.parse(line.strip))
+          end
+        else
+          JSON.parse(@source.read).each(&block)
+        end
+        @source.rewind
+      when Array, Enumerator
+        @source.each(&block)
+      end
+    end
+  end
+end

data/lib/parqueteur/type_resolver.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class TypeResolver
+    def self.resolve(*args)
+      new.resolve(*args)
+    end
+
+    def resolve(type, options = {})
+      case type
+      when :array
+        elements_opt = options.fetch(:elements)
+        Arrow::ListDataType.new(
+          if elements_opt.is_a?(Hash)
+            resolve(elements_opt.fetch(:type), elements_opt)
+          else
+            resolve(elements_opt)
+          end
+        )
+      when :boolean
+        Arrow::BooleanDataType.new
+      when :integer
+        if options.fetch(:unsigned, false) == true
+          Arrow::UInt32DataType.new
+        else
+          Arrow::Int32DataType.new
+        end
+      when :long
+        if options.fetch(:unsigned, false) == true
+          Arrow::UInt64DataType.new
+        else
+          Arrow::Int64DataType.new
+        end
+      when :timestamp
+        Arrow::TimestampDataType.new(
+          options.fetch(:unit, :second)
+        )
+      when :string
+        Arrow::StringDataType.new
+      when :map
+        map_value = options.fetch(:value)
+        Arrow::MapDataType.new(
+          resolve(options.fetch(:key)),
+          if map_value.is_a?(Hash)
+            resolve(map_value.fetch(:type), map_value)
+          else
+            resolve(map_value)
+          end
+        )
+      else
+        raise Error, "unknown type: #{type}"
+      end
+    end
+  end
+end
+
+private
+
+def build_arrow_type(type, options = {})
+
+end

data/lib/parqueteur/value_array_builder.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  class ValueArrayBuilder
+    attr_reader :type, :options, :arrow_type
+
+    def self.build(input, type, options)
+      new(type, options).build(input)
+    end
+
+    def initialize(type, options)
+      @type = type
+      @options = options
+      @arrow_type = Parqueteur::TypeResolver.resolve(type, options)
+    end
+
+    def build(input)
+      return if input.nil?
+
+      case type
+      when :array
+        Arrow::ListArrayBuilder.build(arrow_type, input)
+      when :map
+        builder = Arrow::MapArrayBuilder.new(arrow_type)
+        input.each do |entry|
+          builder.append_value
+          next if entry.nil?
+
+          entry.each do |k, v|
+            builder.key_builder.append(k)
+            builder.item_builder.append(v)
+          end
+        end
+
+        builder.finish
+      when :boolean
+        Arrow::BooleanArray.new(input)
+      when :integer
+        if options.fetch(:unsigned, false) == true
+          Arrow::UInt32Array.new(input)
+        else
+          Arrow::Int32Array.new(input)
+        end
+      when :long
+        if options.fetch(:unsigned, false) == true
+          Arrow::UInt64Array.new(input)
+        else
+          Arrow::Int64Array.new(input)
+        end
+      when :string
+        Arrow::StringArray.new(input)
+      when :timestamp
+        Arrow::TimestampArray.new(input)
+      else
+        raise Error, "unknown type: #{type}"
+      end
+    end
+  end
+end

data/lib/parqueteur/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  VERSION = '1.0.2'
+end

data/lib/parqueteur.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "parqueteur/version"
+require 'parqueteur/type_resolver'
+require 'parqueteur/column'
+require 'parqueteur/column_collection'
+require 'parqueteur/converter'
+require 'parqueteur/chunked_converter'
+require 'parqueteur/input'
+require 'parqueteur/value_array_builder'
+require 'json'
+require 'parquet'
+
+module Parqueteur
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/parqueteur.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "lib/parqueteur/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "parqueteur"
+  spec.version       = Parqueteur::VERSION
+  spec.authors       = ["Julien D."]
+  spec.email         = ["julien@pocketsizesun.com"]
+  spec.license       = 'Apache-2.0'
+  spec.summary       = 'Parqueteur - A Ruby gem that convert JSON to Parquet'
+  spec.description   = 'Convert JSON to Parquet'
+  spec.homepage      = 'https://github.com/pocketsizesun/parqueteur-ruby'
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  # spec.metadata["homepage_uri"] = spec.homepage
+  # spec.metadata["source_code_uri"] = "Put your gem's public repo URL here."
+  # spec.metadata["changelog_uri"] = "Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+  spec.add_dependency 'red-parquet', '~> 5.0'
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/test.json

@@ -0,0 +1 @@
+[{"id":1,"reference":"coucou","hash":{"a":"b"},"valid":true,"hash2":{},"numbers":[1,2,3],"map_array":[]},{"id":2,"reference":"coucou","hash":{"c":"d"},"valid":false,"hash2":{},"numbers":[4,5,6],"map_array":[]},{"id":3,"reference":"coucou","hash":{"e":"f"},"valid":true,"hash2":{"x":[1,2,3]},"numbers":[7,8,9],"map_array":[{"x":"y"}]}]

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: parqueteur
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+platform: ruby
+authors:
+- Julien D.
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-10-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: red-parquet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Convert JSON to Parquet
+email:
+- julien@pocketsizesun.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- example.rb
+- lib/parqueteur.rb
+- lib/parqueteur/chunked_converter.rb
+- lib/parqueteur/column.rb
+- lib/parqueteur/column_collection.rb
+- lib/parqueteur/converter.rb
+- lib/parqueteur/input.rb
+- lib/parqueteur/type_resolver.rb
+- lib/parqueteur/value_array_builder.rb
+- lib/parqueteur/version.rb
+- parqueteur.gemspec
+- test.json
+homepage: https://github.com/pocketsizesun/parqueteur-ruby
+licenses:
+- Apache-2.0
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Parqueteur - A Ruby gem that convert JSON to Parquet
+test_files: []