parqueteur 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86f767a68e38cdd93da015e4ddfcda06b2eefe553ecb6d7a423b4cc0f2752183
-  data.tar.gz: cd741b1d023e44fcc14c08192b9791b675339736527f47dbe2393e85bdff9d07
+  metadata.gz: 6efc312c6b66cee861b45a8384c5f5a7b14e6242c54b1bc3dc02f0c3d1b275ad
+  data.tar.gz: 633eeed8b83bbbb277672a30920052f8c9d48189896eb9ec4418d8b12c5fc179
 SHA512:
-  metadata.gz: 58882a4d2d1ea5a0cb53f5643f96ac504352b87e6c1436d10841a616d2867bf127c70a31374f6ff2f455fb2894d29fe8cc439ca8a0efca5fe519c00ee312c8c5
-  data.tar.gz: ac185e758d8c0fa19d11ac05e96f77f93cb8260e055d203d1d0d71dd04e0ff6db70d773824a8a6fe5c63e5f235ab179f3417ac689b590403b69192cbad0bde98
+  metadata.gz: 26b4113e8740452055413f25260179ff9688e4fc5cac882e1a192e646e2a2635b7fe76393032a37656103dcf797da22ad4f269e18db64f9e8d5ab0dd94375d02
+  data.tar.gz: 95e293e1233d247cdb43a3ed510c5f5a2e5bc35511260221cb4c2e55de0ecdde12607d1bd63fac6e12e7481f20ffea017526bf0650c500dcfec408a4c29a1fa3

data/Gemfile.lock

@@ -1,32 +1,34 @@
 PATH
   remote: .
   specs:
-    parqueteur (1.3.1)
-      red-parquet (~> 5.0)
+    parqueteur (1.4.0)
+      red-parquet (~> 7.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    bigdecimal (3.0.2)
+    bigdecimal (3.1.2)
     extpp (0.1.0)
-    gio2 (3.4.9)
-      gobject-introspection (= 3.4.9)
-    glib2 (3.4.9)
+    fiddle (1.1.0)
+    gio2 (3.5.1)
+      fiddle
+      gobject-introspection (= 3.5.1)
+    glib2 (3.5.1)
       native-package-installer (>= 1.0.3)
       pkg-config (>= 1.3.5)
-    gobject-introspection (3.4.9)
-      glib2 (= 3.4.9)
-    native-package-installer (1.1.1)
-    pkg-config (1.4.6)
+    gobject-introspection (3.5.1)
+      glib2 (= 3.5.1)
+    native-package-installer (1.1.4)
+    pkg-config (1.4.7)
     rake (13.0.6)
-    red-arrow (5.0.0)
+    red-arrow (7.0.0)
       bigdecimal (>= 2.0.3)
       extpp (>= 0.0.7)
-      gio2 (>= 3.4.5)
+      gio2 (>= 3.5.0)
       native-package-installer
       pkg-config
-    red-parquet (5.0.0)
-      red-arrow (= 5.0.0)
+    red-parquet (7.0.0)
+      red-arrow (= 7.0.0)
 
 PLATFORMS
   ruby

data/examples/hello-world.rb

@@ -1,3 +1,4 @@
+require 'securerandom'
 require 'bundler/setup'
 require 'parqueteur'
 
@@ -17,14 +18,16 @@ class FooParquetConverter < Parqueteur::Converter
   column :my_timestamp, :timestamp
 end
 
-data = 100.times.collect do |i|
+data = 1000.times.collect do |i|
   {
     'id' => i,
     'my_string_array' => %w[a b c],
     'my_date' => Date.today,
     'my_decimal' => BigDecimal('0.03'),
     'my_int' => rand(1..10),
-    'my_map' => { 'a' => 'b' },
+    'my_map' => 20.times.each_with_object({}) do |idx, hash|
+      hash["k_#{idx}"] = SecureRandom.urlsafe_base64(64)
+    end,
     'my_string' => 'Hello World',
     'my_struct' => {
       'my_struct_str' => 'Hello World',

data/lib/parqueteur/converter.rb

@@ -2,7 +2,7 @@
 
 module Parqueteur
   class Converter
-    DEFAULT_BATCH_SIZE = 10
+    DEFAULT_BATCH_SIZE = 100
 
     def self.inline(&block)
       Class.new(self, &block)

data/lib/parqueteur/types/map_type.rb

@@ -4,18 +4,7 @@ module Parqueteur
   module Types
     class MapType < Parqueteur::Type
       def build_value_array(values)
-        builder = Arrow::MapArrayBuilder.new(arrow_type)
-        values.each do |entry|
-          builder.append_value
-          next if entry.nil?
-
-          entry.each do |k, v|
-            builder.key_builder.append(k)
-            builder.item_builder.append(v)
-          end
-        end
-
-        builder.finish
+        Arrow::MapArrayBuilder.build(arrow_type, values)
       end
 
       def arrow_type_builder

data/lib/parqueteur/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parqueteur
-  VERSION = '1.3.1'
+  VERSION = '1.4.0'
 end

data/parqueteur.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
 
   # Uncomment to register a new dependency of your gem
   # spec.add_dependency "example-gem", "~> 1.0"
-  spec.add_dependency 'red-parquet', '~> 5.0'
+  spec.add_dependency 'red-parquet', '~> 7.0'
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parqueteur
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Julien D.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-17 00:00:00.000000000 Z
+date: 2022-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-parquet
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '7.0'
 description: Convert data to Parquet
 email:
 - julien@pocketsizesun.com
@@ -91,7 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Parqueteur - A Ruby gem that convert data to Parquet