metacrunch 4.1.1 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c57943da0f6fc1236b58f06326223070fd1cf20d
-  data.tar.gz: 0a905216b6a340aed1fe95425711877cfd3384ea
+  metadata.gz: 1fc2560b2bb768757c384d71a806509d594e7610
+  data.tar.gz: 8cbb0e384582550d29f94daa2d7521942cbe14c9
 SHA512:
-  metadata.gz: '086cafcc6d00451da522d74a57379fa9be25538b77f4733be5ee3a1923ea38933394e00dd111fdbdfba43055989db2ba0e2879b5145ae3aa7fb8060e8f6295db'
-  data.tar.gz: 19c2bfaf59bd1e695aa5569f12037448ce19a52cde3c93a62b9afa99b3c4269fc5efba1042b7321d255c099ae8b44cff269d25ad9b14b44928f9983badfae61b
+  metadata.gz: eaf6d9b6b72b7cadc92dae0d4d9e1204f6d43fe909f2f5f60bda85ad0407aa6fffe567a119361d3cef3fa15ed2d83ca5abc44d314eafc3f59b7b936c49a07ab5
+  data.tar.gz: 0204e2e3284a53c007ea086b883ff52fbb87e0731616fe83c1c4d636134985506c39d885ec8ff4bc4cc8708e3a51d02c27e28fe79b732798f7f3bfc24905ac6d

data/Readme.md

@@ -72,7 +72,10 @@ To process, transform or manipulate data use the `#transformation` hook. A trans
 
 The current data object (the last object yielded by the source) will be passed to the first transformation as a parameter. The return value of a transformation will then be passed to the next transformation and so on.
 
-If you return `nil` the current data object will be dismissed and the next transformation won't be called.
+There are two exceptions to that rule.
+
+* If you return `nil` the current data object will be dismissed and the next transformation won't be called.
+* If you return an `Enumerator` the object will be expanded and the following transformations will be called with each element of the `Enumerator`.
 
 ```ruby
 # File: my_etl_job.metacrunch
@@ -119,7 +122,7 @@ transformation ->(bulk) {
 
 # A buffer that uses a Proc
 transformation ->(bulk) { 
-  # ...
+  # Called when the buffer `Proc` returns `true`
 }, buffer: -> {
   true if some_condition
 }

data/lib/metacrunch/job.rb

@@ -83,12 +83,12 @@ class Metacrunch::Job # http://valve.github.io/blog/2013/10/26/constant-resoluti
     if source
       # Run transformation for each data object available in source
       source.each do |data|
-        data = run_transformations(data)
+        data = run_transformations(transformations, data)
         write_destination(data)
       end
 
       # Run all transformations a last time to flush existing buffers
-      data = run_transformations(nil, flush_buffers: true)
+      data = run_transformations(transformations, nil, flush_buffers: true)
       write_destination(data)
 
       # Close destination
@@ -123,19 +123,18 @@ private
     post_process.call if post_process
   end
 
-  def run_transformations(data, flush_buffers: false)
-    transformations.each do |transformation|
+  def run_transformations(transformations, data, flush_buffers: false)
+    transformations.each.with_index do |transformation, i|
       if transformation.is_a?(Buffer)
-        buffer = transformation
-
-        if data
-          data = buffer.buffer(data)
-          data = buffer.flush if flush_buffers
-        else
-          data = buffer.flush
-        end
+        data = transformation.buffer(data) if data
+        data = transformation.flush if flush_buffers
       else
         data = transformation.call(data) if data
+
+        if data&.is_a?(Enumerator)
+          data.each { |d| run_transformations(transformations.slice(i+1..-1), d, flush_buffers: flush_buffers) }
+          data = nil
+        end
       end
     end
 
@@ -143,7 +142,7 @@ private
   end
 
   def write_destination(data)
-    destination.write(data) if destination && data
+    destination.write(data) if destination && data.present?
   end
 
 end

data/lib/metacrunch/job/buffer.rb

@@ -22,7 +22,7 @@ module Metacrunch
     end
 
     def flush
-      @buffer
+      @buffer.presence
     ensure
       @buffer = []
     end

data/lib/metacrunch/version.rb

@@ -1,3 +1,3 @@
 module Metacrunch
-  VERSION = "4.1.1"
+  VERSION = "4.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.2.0
 platform: ruby
 authors:
 - René Sprotte