parqueteur 1.0.3 → 1.3.0

data/lib/parqueteur/types/time32_type.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Time32Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Time32Array.new(
+          @options.fetch(:precision, :second), values
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::Time32DataType.new(
+          options.fetch(:unit, :second)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/time64_type.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Parqueteur
+  module Types
+    class Time64Type < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::Time64Array.new(
+          @options.fetch(:precision, :second), values
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::Time64DataType.new(
+          options.fetch(:unit, :second)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/types/timestamp_type.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# when :timestamp
+#   Arrow::TimestampDataType.new(
+#     options.fetch(:unit, :second)
+#   )
+
+module Parqueteur
+  module Types
+    class TimestampType < Parqueteur::Type
+      def build_value_array(values)
+        Arrow::TimestampArray.new(
+          options.fetch(:unit, :second), values
+        )
+      end
+
+      def arrow_type_builder
+        Arrow::TimestampDataType.new(
+          options.fetch(:unit, :second)
+        )
+      end
+    end
+  end
+end

data/lib/parqueteur/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parqueteur
-  VERSION = '1.0.3'
+  VERSION = '1.3.0'
 end

data/lib/parqueteur.rb

@@ -1,17 +1,34 @@
 # frozen_string_literal: true
 
-require_relative "parqueteur/version"
-require 'parqueteur/type_resolver'
+require 'json'
+require 'singleton'
+require 'tempfile'
+require 'parquet'
+
+require_relative 'parqueteur/version'
 require 'parqueteur/column'
 require 'parqueteur/column_collection'
 require 'parqueteur/converter'
-require 'parqueteur/chunked_converter'
 require 'parqueteur/input'
-require 'parqueteur/value_array_builder'
-require 'json'
-require 'parquet'
+require 'parqueteur/struct'
+require 'parqueteur/type'
+require 'parqueteur/type_resolver'
+require 'parqueteur/types/array_type'
+require 'parqueteur/types/boolean_type'
+require 'parqueteur/types/date32_type'
+require 'parqueteur/types/date64_type'
+require 'parqueteur/types/decimal128_type'
+require 'parqueteur/types/decimal256_type'
+require 'parqueteur/types/int32_type'
+require 'parqueteur/types/int64_type'
+require 'parqueteur/types/map_type'
+require 'parqueteur/types/string_type'
+require 'parqueteur/types/struct_type'
+require 'parqueteur/types/time32_type'
+require 'parqueteur/types/time64_type'
+require 'parqueteur/types/timestamp_type'
 
 module Parqueteur
   class Error < StandardError; end
-  # Your code goes here...
+  class TypeNotFound < Error; end
 end

data/parqueteur.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Julien D."]
   spec.email         = ["julien@pocketsizesun.com"]
   spec.license       = 'Apache-2.0'
-  spec.summary       = 'Parqueteur - A Ruby gem that convert JSON to Parquet'
-  spec.description   = 'Convert JSON to Parquet'
+  spec.summary       = 'Parqueteur - A Ruby gem that convert data to Parquet'
+  spec.description   = 'Convert data to Parquet'
   spec.homepage      = 'https://github.com/pocketsizesun/parqueteur-ruby'
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 

data/scripts/apache-arrow-ubuntu-install.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+if [ $(dpkg-query -W -f='${Status}' apache-arrow-apt-source 2>/dev/null | grep -c "ok installed") -eq 1 ]
+then
+  exit 0
+fi
+
+LSB_RELEASE_CODENAME_SHORT=$(lsb_release --codename --short)
+
+apt-get update
+apt-get install -y -V ca-certificates lsb-release wget
+wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+apt-get install -y -V ./apache-arrow-apt-source-latest-${LSB_RELEASE_CODENAME_SHORT}.deb
+rm ./apache-arrow-apt-source-latest-${LSB_RELEASE_CODENAME_SHORT}.deb
+apt-get update
+apt-get install -y libgirepository1.0-dev libarrow-dev libarrow-glib-dev libparquet-dev libparquet-glib-dev
+
+exit 0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parqueteur
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.3.0
 platform: ruby
 authors:
 - Julien D.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-02 00:00:00.000000000 Z
+date: 2021-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-parquet
@@ -24,7 +24,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-description: Convert JSON to Parquet
+description: Convert data to Parquet
 email:
 - julien@pocketsizesun.com
 executables: []
@@ -38,18 +38,37 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- example.rb
+- examples/convert-and-compression.rb
+- examples/convert-methods.rb
+- examples/convert-to-io.rb
+- examples/convert-with-chunks.rb
+- examples/convert-without-compression.rb
+- examples/hello-world.rb
 - lib/parqueteur.rb
-- lib/parqueteur/chunked_converter.rb
 - lib/parqueteur/column.rb
 - lib/parqueteur/column_collection.rb
 - lib/parqueteur/converter.rb
 - lib/parqueteur/input.rb
+- lib/parqueteur/struct.rb
+- lib/parqueteur/type.rb
 - lib/parqueteur/type_resolver.rb
-- lib/parqueteur/value_array_builder.rb
+- lib/parqueteur/types/array_type.rb
+- lib/parqueteur/types/boolean_type.rb
+- lib/parqueteur/types/date32_type.rb
+- lib/parqueteur/types/date64_type.rb
+- lib/parqueteur/types/decimal128_type.rb
+- lib/parqueteur/types/decimal256_type.rb
+- lib/parqueteur/types/int32_type.rb
+- lib/parqueteur/types/int64_type.rb
+- lib/parqueteur/types/map_type.rb
+- lib/parqueteur/types/string_type.rb
+- lib/parqueteur/types/struct_type.rb
+- lib/parqueteur/types/time32_type.rb
+- lib/parqueteur/types/time64_type.rb
+- lib/parqueteur/types/timestamp_type.rb
 - lib/parqueteur/version.rb
 - parqueteur.gemspec
-- test.json
+- scripts/apache-arrow-ubuntu-install.sh
 homepage: https://github.com/pocketsizesun/parqueteur-ruby
 licenses:
 - Apache-2.0
@@ -73,5 +92,5 @@ requirements: []
 rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
-summary: Parqueteur - A Ruby gem that convert JSON to Parquet
+summary: Parqueteur - A Ruby gem that convert data to Parquet
 test_files: []

data/example.rb

@@ -1,20 +0,0 @@
-require 'bundler/setup'
-require 'parqueteur'
-
-class Foo < Parqueteur::Converter
-  column :id, :long
-  column :reference, :string
-  column :hash, :map, key: :string, value: :string
-  column :valid, :boolean
-  column :total, :integer
-end
-
-LETTERS = ('a'..'z').to_a
-
-data = 1000.times.collect do |i|
-  { 'id' => i + 1, 'reference' => "coucou:#{i}", 'hash' => { 'a' => LETTERS.sample }, 'valid' => rand < 0.5, 'total' => rand(100..500) }
-end
-
-chunked_converter = Parqueteur::ChunkedConverter.new(data, Foo)
-pp chunked_converter.write_files('test')
-# puts Foo.convert(data, output: 'test.parquet')

data/lib/parqueteur/chunked_converter.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Parqueteur
-  class ChunkedConverter
-    attr_reader :schema
-
-    def initialize(input, converter, chunk_size = 200)
-      @input = Parqueteur::Input.from(input)
-      @converter = converter
-      @chunk_size = chunk_size
-    end
-
-    def chunks
-      Enumerator.new do |arr|
-        @input.each_slice(@chunk_size) do |chunk|
-          local_converter = @converter.new(chunk)
-          arr << local_converter.to_io
-        end
-      end
-    end
-
-    def write_files(prefix)
-      chunks.each_with_index do |chunk, idx|
-        File.write("#{prefix}.#{idx}.parquet", chunk.read)
-      end
-    end
-  end
-end

data/lib/parqueteur/value_array_builder.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module Parqueteur
-  class ValueArrayBuilder
-    attr_reader :type, :options, :arrow_type
-
-    def self.build(input, type, options)
-      new(type, options).build(input)
-    end
-
-    def initialize(type, options)
-      @type = type
-      @options = options
-      @arrow_type = Parqueteur::TypeResolver.resolve(type, options)
-    end
-
-    def build(input)
-      return if input.nil?
-
-      case type
-      when :array
-        Arrow::ListArrayBuilder.build(arrow_type, input)
-      when :map
-        builder = Arrow::MapArrayBuilder.new(arrow_type)
-        input.each do |entry|
-          builder.append_value
-          next if entry.nil?
-
-          entry.each do |k, v|
-            builder.key_builder.append(k)
-            builder.item_builder.append(v)
-          end
-        end
-
-        builder.finish
-      when :boolean
-        Arrow::BooleanArray.new(input)
-      when :integer
-        if options.fetch(:unsigned, false) == true
-          Arrow::UInt32Array.new(input)
-        else
-          Arrow::Int32Array.new(input)
-        end
-      when :long
-        if options.fetch(:unsigned, false) == true
-          Arrow::UInt64Array.new(input)
-        else
-          Arrow::Int64Array.new(input)
-        end
-      when :string
-        Arrow::StringArray.new(input)
-      when :timestamp
-        Arrow::TimestampArray.new(input)
-      else
-        raise Error, "unknown type: #{type}"
-      end
-    end
-  end
-end

data/test.json

@@ -1 +0,0 @@
-[{"id":1,"reference":"coucou","hash":{"a":"b"},"valid":true,"hash2":{},"numbers":[1,2,3],"map_array":[]},{"id":2,"reference":"coucou","hash":{"c":"d"},"valid":false,"hash2":{},"numbers":[4,5,6],"map_array":[]},{"id":3,"reference":"coucou","hash":{"e":"f"},"valid":true,"hash2":{"x":[1,2,3]},"numbers":[7,8,9],"map_array":[{"x":"y"}]}]